def post(self): user = self.getAuthentificatedUser() if not user: return try: id = int(self.request.get('id')) topic = Topic().get(db.Key.from_path('Topic', id)) except: self.redirect('/') return post = Post(parent=topic.key()) post.topic = topic if users.get_current_user(): post.author = users.get_current_user() body = db.Text(strip_ml_tags(self.request.get('body'))) postmarkup = create(use_pygments=False) post.body = postmarkup(body) # replace('\n','<br />') if post.body != '': post.put() #mailAdditionalText = """ ... testing e-mail notification. Sorry if you get this message accidently.""" #post.sendMailToAll(user.email(), mailAdditionalText) ##### #message = mail.EmailMessage(sender=user.email(), subject="New message in small-forum") #message.to = "log1 (sms) <*****@*****.**>" #message.body = post.body #message.send() ##### # To Do if self.request.get('page'): self.redirect('/view?id=' + str(self.request.get('id')) + '&page=' + self.request.get('page')) else: self.redirect('/view?id=' + str(self.request.get('id')))
def post(self): user = self.getAuthentificatedUser() if not user: return try: id = int(self.request.get('id')) topic = Topic().get(db.Key.from_path('Topic', id)) except: self.redirect('/') return post = Post(parent=topic.key()) post.topic = topic if users.get_current_user(): post.author = users.get_current_user() body = db.Text(strip_ml_tags(self.request.get('body'))) postmarkup = create(use_pygments=False) post.body = postmarkup(body) # replace('\n','<br />') if post.body != '': post.put() message = mail.EmailMessage(sender=user.email(), subject="New message in small-forum") message.to = "log1 <*****@*****.**>" message.body = post.body + """ ... testing e-mail notification. Sorry if you get this message accidently.""" message.send() #topic = Topic().all().filter('id =', int(self.request.get('id'))) #topic = topic[0] #topic.getUserList() # To Do if self.request.get('page'): self.redirect('/view?id=' + str(self.request.get('id')) + '&page=' + self.request.get('page')) else: self.redirect('/view?id=' + str(self.request.get('id')))
async def login(): #TODO 从配置文件读账号密码 payload = { 'client_id': '0ab215a8b1977939201640fa14c66bab', 'grant_type': 'password', 'client_secret': '22b2cf86ccc81009', 'username': USERNAME, 'password': PASSWORD } async with aiohttp.ClientSession() as session: #拿调用api的token async with session.post(url=login_url, headers=headers, data=payload) as r: json_body = await r.json() headers['Authorization'] = 'Bearer ' + json_body['access_token'] #调group最新的tpoics async with session.get(url=group_url, headers=headers, data={'count': 100}) as r: json_body = await r.json() for topic in json_body['topics']: print(topic) t = Topic() t.update_time = topic['update_time'] t.title = topic['title'] t.url = topic['url'] t.create_time = topic['create_time'] t.comments_count = topic['comments_count'] t.meta.id = topic['id'] t.save() print(topic['url'])
def __init__(self, loop=None): self.loop = loop or asyncio.get_event_loop() self.root = 'https://frodo.douban.com/api/v2/group/146409/topics?count=100&start={offset}' self.seen_urls = set() self.done = [] self.q = asyncio.Queue(loop=self.loop) self.session = aiohttp.ClientSession(loop=self.loop) self.headers = {'User-Agent': 'api-client/0.1.3 com.douban.frodo/5.20.0 iOS/11.2.5 iPhone8,4 network/wifi'} self.t0 = time.time() self.t1 = None self.count = 0 self.topic = Topic()
def addNewTopic(): """Gets the information obtained from the form and process it for adding to the timeline""" topic_title = request.form.get("topic_title") topic_day = request.form.get("topic_day") topic_month = request.form.get("topic_month") topic_year = request.form.get("topic_year") topic_lat = request.form.get("topic_lat") topic_lng = request.form.get("topic_lng") topic_image = request.form.get("topic_image") topic_desc = request.form.get("topic_desc") date = datetime(int(topic_year), int(topic_month), int(topic_day)) new_topic = Topic(center_lat=topic_lat, center_lng=topic_lng, description=topic_desc, main_date=date, topic_title=topic_title, image=topic_image) print "TOPIC TO ADD:", new_topic db.session.add(new_topic) db.session.commit() #Adds the new topic as an event topic_id = db.session.query(Topic.topic_id).filter(Topic.description ==topic_desc, Topic.main_date==date, Topic.topic_title==topic_title).one() new_event = Event_data(topic_id=topic_id[0], lat=topic_lat, lng=topic_lng, description=topic_desc, event_date=date, event_title=topic_title, image=topic_image ) db.session.add(new_event) db.session.commit() return redirect('/')
def post(self): topic = Topic.get_by_id(int(self.request.get('id'))) if not topic: self.response.out.write('莫有找到台词~~~') return user = users.get_current_user() if topic.answer == self.request.get('answer'): if user: ua = UserAnswer() ua.user = user ua.topic = topic ua.put() uac = UserAnswerCount() uac.add(user) memcache.delete('topic_list::latest' + user.user_id()) if not topic.answered_count: topic.answered_count = 1 else: topic.answered_count += 1 topic.put() self.response.out.write(json_output('ok', {'message': '答对了,不错哦'})) return else: if not topic.failed_count: topic.failed_count = 1 else: topic.failed_count += 1 topic.put() self.response.out.write(json_output('fail', {'message': '再想想?'}))
class PostForm(FlaskForm): title = StringField('Title', validators=[DataRequired()]) content = TextAreaField('Content', validators=[DataRequired()]) topic = SelectField('Topic', choices=[(t.tid, t.topic) for t in Topic.all_topics()], coerce=int) submit = SubmitField('Post')
def post(self): user = users.get_current_user() if not user: self.response.out.write('~!@#$%') return topic = Topic.get(self.request.get('entity')) if int(self.request.get('mark')) != 1: if not topic.votedown: topic.votedown = -1 else: topic.votedown -= 1 else: if not topic.voteup: topic.voteup = 1 else: topic.voteup += 1 topic.put() vote = Vote() vote.author = users.get_current_user() vote.topic = topic vote.mark = int(self.request.get('mark')) vote.put() memcache.delete('topic_list::latest' + user.user_id()) self.response.out.write(json_output('ok'))
def get(self, topic_id): full_topic_id = Topic.get_long_uid(topic_id) topic = self.api.get_by_id(full_topic_id) # check all content objects were found if topic is not None: # Increment view counts on the topic view_counter.increment(full_topic_id) # find lessons in topic lessons = [] if topic.lessons: lessons = self.api.get_by_id(topic.lessons) # Get related practices related_practices = Practice.get_related_practices(topic, 6) self.write( 'topic.html', topic=topic, lessons=lessons, color=topic.color, related_practices=related_practices, ) else: # 404 if topic cannot be found return self.http_not_found()
def get(self): forum = Forum.from_url(self.request.path_info) user = users.get_current_user() is_admin = users.is_current_user_admin() if not forum or (forum.is_disabled and not is_admin): return self.redirect("/") if not is_admin and not self.settings.check_ip(self.request.remote_addr): return self.response.out.write('Your IP address has been banned') if not is_admin and not self.settings.check_user( user ): return self.redirect( users.create_login_url("/") ) offset, topics = Topic.getlist(forum, is_admin=is_admin, offset=self.request.get("from") or None) for topic in topics: topic.excerpt = Post.gql("WHERE topic = :1 ORDER BY created_on", topic)[0].get_excerpt() tvals = { 'user': user, 'analytics_code': forum.analytics_code or "", 'siteurl': self.request.url, 'isadmin': is_admin, 'forum' : forum, 'forum_urls': [f.url for f in Forum.all()], 'topics': topics, 'offset': offset, 'login_url': users.create_login_url(forum.root()), 'logout_url': users.create_logout_url(forum.root()) } self.template_out("skins/default/topic_list.html", tvals)
def create_topic(topic_category): """Create and return a topic by the topic name.""" topic = Topic(topic_category=topic_category) db.session.add(topic) db.session.commit()
def post(self): user = users.get_current_user() if not user: self.response.out.write('~!@#$%') return topic = Topic.get(self.request.get('entity')) if int(self.request.get('mark')) != 1: if not topic.votedown: topic.votedown = -1 else: topic.votedown -= 1 else: if not topic.voteup: topic.voteup = 1 else: topic.voteup += 1 topic.put() vote = Vote() vote.author = users.get_current_user() vote.topic = topic vote.mark = int(self.request.get('mark')) vote.put() memcache.delete('topic_list::latest'+user.user_id()) self.response.out.write(json_output('ok'))
def get_render_topic_list(self, list_type='latest'): page = int(self.request.get('page', 1)) user = users.get_current_user() user_key = user.user_id() if user else 'guest' topic_list = memcache.get('topic_list::' + list_type + '::' + user_key) if not topic_list or page != 1: offset = (page - 1) * ITEMS_PER_PAGE topic = Topic.all() if list_type == 'latest': topic.order('-date') elif list_type == 'top': topic.order('-voteup') topic.order('votedown') topic_list = topic.fetch(limit=ITEMS_PER_PAGE, offset=offset) if page == 1: if type == 'top': memcache.set('topic_list::top::' + user_key, topic_list, 1800) else: memcache.set('topic_list::latest::' + user_key, topic_list, 60) return self.get_render('topic_list', { 'list': topic_list, 'list_length': len(topic_list), 'page': page + 1 })
def post(self): topic = Topic.get_by_id(int(self.request.get('id'))) if not topic: self.response.out.write('莫有找到台词~~~') return user = users.get_current_user() if topic.answer == self.request.get('answer'): if user: ua = UserAnswer() ua.user = user ua.topic = topic ua.put() uac = UserAnswerCount() uac.add(user) memcache.delete('topic_list::latest'+user.user_id()) if not topic.answered_count: topic.answered_count = 1 else: topic.answered_count += 1 topic.put() self.response.out.write(json_output('ok', {'message': '答对了,不错哦'})) return else: if not topic.failed_count: topic.failed_count = 1 else: topic.failed_count += 1 topic.put() self.response.out.write(json_output('fail', {'message': '再想想?'}))
def post(self): args = parser.parse_args() try: print(args) modified = Topic.create(args) return dict(success=modified > 0) except Exception as e: return dict(success=False, message=str(e))
def post(self): user = self.getAuthentificatedUser() if not user: return id = self.request.get('id') try: thread = Thread.get(db.Key.from_path('Thread', int(id))) except: return name = strip_ml_tags(self.request.get('name')) if name == '': template_values = { 'topics' : self.topics, 'name' : name, } else: topic = Topic() #parent=thread topic.thread = thread topic.name = name if users.get_current_user(): topic.author = users.get_current_user() topic.put() mode = self.request.get('mode') self.redirect('/view?id=' + str(topic.key().id())) return template_values = { 'topics' : self.topics, 'name' : '', } path = os.path.join(os.path.dirname(__file__), os.path.join('templates', 'addTopic.htm')) self.response.out.write(template.render(path, template_values))
def write_topic_perWindow(): time_window,topics = yield print "write_topic_perWindow received: \n" + time_window + "\n" + topics DBSession = sessionmaker(bind=engine) session = DBSession() topics = Topic(time_window = time_window,topics = topics) session.merge(topics) session.commit() session.flush()
def get(self, nodeid, topicid): t_key = 't-%s-%s' % (str(nodeid), str(topicid)) t_obj = Topic.get_by_key(t_key) if not t_obj or t_obj.has_key('hide'): self.set_status(404) self.echo( 'error.html', { 'page': '404', 'title': "Can't find out this URL", 'h2': 'Oh, my god!', 'msg': 'Something seems to be lost...' }) return if t_obj['cnum']: cnum = int(t_obj['cnum']) from_id = int(self.get_argument('id', '1')) if from_id > 1 and from_id % EACH_PAGE_COMMENT_NUM != 1: self.redirect('/' + t_key) return to_id = from_id + EACH_PAGE_COMMENT_NUM - 1 if to_id > cnum: to_id = cnum c_objs = Comment.get_comments(t_key, from_id, to_id) else: c_objs = None from_id = to_id = cnum = 0 self.echo('topicdetail.html', { 'title': t_obj['title'] + ' - ' + t_obj['nodename'], 'description': 'description', 't_obj': t_obj, 'c_objs': c_objs, 'from_id': from_id, 'to_id': to_id, 'cnum': cnum, 'newest_node': Node.get_newest(), 'recent_node': Node.get_recent_node(), 'hot_node': Node.get_hot_node(), 'recent_topic_objs': Commomkvdb.get_comment_topic_by_keys('recent-topic-home'), 'comment_topic_objs': Commomkvdb.get_comment_topic_by_keys('recent-comment-topic-home'), }, layout='_layout.html')
def get(self): if not self.checkIfAuthentificatedUserIsAdmin(): return try: id = self.request.get('id') topic = Topic.get(db.Key.from_path('Topic', int(id))) thread_id = topic.thread.key().id() topic.delete() except: pass self.redirect('/viewThread?mode=admin&id='+str(thread_id))
def get(self): t_key = str(self.get_argument('key','')) if t_key: t_obj = Topic.get_by_key(t_key) else: t_obj = TOPIC_DICT.copy() self.echo('editpost.html', { 'title': "修改帖子", 't_key': t_key, 't_obj': t_obj, }, layout='_layout.html')
def get(self, nodeid, topicid): topic_key = '%s-%s' % (nodeid, topicid) t_obj = Topic.get_by_key_name(topic_key) if not t_obj: self.error(404) self.echo( 'error.html', { 'page': '404', 'title': "Can't find out this URL", 'h2': 'Oh, my god!', 'msg': 'Something seems to be lost...' }) return if t_obj.cnum: from_id = int(self.request.get('id', '1')) if from_id > 1 and from_id % EACH_PAGE_COMMENT_NUM != 1: self.redirect('/t-' + topic_key) return to_id = from_id + EACH_PAGE_COMMENT_NUM - 1 if to_id > t_obj.cnum: to_id = t_obj.cnum c_objs = Comment.get_comments(topic_key, from_id, to_id) else: c_objs = None from_id = to_id = cnum = 0 self.echo( 'topicdetail.html', { 'title': t_obj.title + ' - ' + t_obj.node.name, 'description': 'description', 't_obj': t_obj, 'c_objs': c_objs, 'from_id': from_id, 'to_id': to_id, 'newest_node': Node.get_newest(), #'recent_node': Node.get_recent_node(), 'hot_node': Node.get_hot_node(), 'recent_topic_objs': KeyStrValue.get_topic_key_title('recent-topic-home'), 'reply_topic_objs': KeyStrValue.get_topic_key_title('recent-reply-topic'), }, layout='_layout.html')
def get(self): t_key = str(self.get_argument('key', '')) if t_key: t_obj = Topic.get_by_key(t_key) else: t_obj = TOPIC_DICT.copy() self.echo('editpost.html', { 'title': "修改帖子", 't_key': t_key, 't_obj': t_obj, }, layout='_layout.html')
def post(self): user = users.get_current_user() if not user: self.response.out.write('~!@#$%') return if (not self.request.get('sentence')): self.response.out.write(json_output('error', { 'message': 'empty sentence', })) return if (not self.request.get('answer')): self.response.out.write(json_output('error', { 'message': 'empty answer', })) return topic = Topic() topic.author = users.get_current_user() topic.sentence = self.request.get('sentence')[:300] topic.answer = self.request.get('answer') topic.put() memcache.delete('topic_list::latest'+user.user_id()) if self.is_ajax: self.response.out.write(json_output('ok', { 'user': topic.author.nickname(), 'sentence': topic.sentence, 'answer': topic.answer, })) else: self.redirect('/')
def get(self): if self.cur_user and self.cur_user.flag==99: t_key = self.request.get('key') if t_key: t_obj = Topic.get_by_key_name(t_key) else: t_obj = None self.echo('editpost.html', { 'title': "修改帖子", 't_obj': t_obj, }, layout='_layout.html') else: self.error(403) self.write('403:forbidden')
def get_topics(self, forum, is_moderator, max_topics, cursor): # note: building query manually beccause gql() don't work with cursor # see: http://code.google.com/p/googleappengine/issues/detail?id=2757 q = Topic.all() q.filter("forum =", forum) if not is_moderator: q.filter("is_deleted =", False) q.order("-created_on") if not cursor is None: q.with_cursor(cursor) topics = q.fetch(max_topics) new_cursor = q.cursor() if len(topics) < max_topics: new_cursor = None return (new_cursor, topics)
def get(self): topic = Topic.get_by_id(int(self.request.get('id'))) if not topic: self.response.out.write('莫有找到台词~~~') return user = users.get_current_user() if user: ua = UserSeen() ua.user = user ua.topic = topic ua.put() memcache.delete('topic_list::latest'+user.user_id()) self.response.out.write(json_output('ok', {'message': topic.answer})) return self.response.out.write('登录后就能看到答案了');
def interview_topics(): """Load interview topics from topics.txt into database.""" print("Loading topics") # Delete all rows in table, so if we need to run this a second time, # we won't be trying to add duplicate questions Topic.query.delete() # Read the file and insert data one by one for row in open("data/topics.txt"): row = row.rstrip() topic = Topic(topic=row) db.session.add(topic) db.session.commit()
def post(self): if self.cur_user and self.cur_user.flag == 99: t_key = self.request.get('key') title = self.POST['title'] content = self.POST['content'] if t_key and title and content: t_obj = Topic.get_by_key_name(t_key) if t_obj: t_obj.title = title t_obj.con = textilize(content) db.run_in_transaction(obj_runput, t_obj) self.redirect('/edit-post?key=%s' % str(t_key)) else: self.error(403) self.write('403:forbidden')
def post(self): if self.cur_user and self.cur_user.flag==99: t_key = self.request.get('key') title = self.POST['title'] content = self.POST['content'] if t_key and title and content: t_obj = Topic.get_by_key_name(t_key) if t_obj: t_obj.title = title t_obj.con = textilize(content) db.run_in_transaction(obj_runput, t_obj) self.redirect('/edit-post?key=%s'%str(t_key)) else: self.error(403) self.write('403:forbidden')
def get(self): topic = Topic.get_by_id(int(self.request.get('id'))) if not topic: self.response.out.write('莫有找到台词~~~') return user = users.get_current_user() if user: ua = UserSeen() ua.user = user ua.topic = topic ua.put() memcache.delete('topic_list::latest' + user.user_id()) self.response.out.write( json_output('ok', {'message': topic.answer})) return self.response.out.write('登录后就能看到答案了')
def get(self, nodeid, topicid): t_key = 't-%s-%s' % (str(nodeid), str(topicid)) t_obj = Topic.get_by_key(t_key) if not t_obj or t_obj.has_key('hide'): self.set_status(404) self.echo('error.html', { 'page': '404', 'title': "Can't find out this URL", 'h2': 'Oh, my god!', 'msg': 'Something seems to be lost...' }) return if t_obj['cnum']: cnum = int(t_obj['cnum']) from_id = int(self.get_argument('id', '1')) if from_id>1 and from_id%EACH_PAGE_COMMENT_NUM!=1: self.redirect('/'+t_key) return to_id = from_id + EACH_PAGE_COMMENT_NUM - 1 if to_id > cnum: to_id = cnum c_objs = Comment.get_comments(t_key, from_id, to_id) else: c_objs = None from_id = to_id = cnum = 0 self.echo('topicdetail.html', { 'title': t_obj['title'] + ' - ' + t_obj['nodename'], 'description':'description', 't_obj': t_obj, 'c_objs': c_objs, 'from_id': from_id, 'to_id': to_id, 'cnum': cnum, 'newest_node': Node.get_newest(), 'recent_node': Node.get_recent_node(), 'hot_node': Node.get_hot_node(), 'recent_topic_objs': Commomkvdb.get_comment_topic_by_keys('recent-topic-home'), 'comment_topic_objs': Commomkvdb.get_comment_topic_by_keys('recent-comment-topic-home'), }, layout='_layout.html')
def post(self): t_key = str(self.get_argument('key', '')) title = self.get_argument('title', '') content = self.get_argument('content', '') hide = self.get_argument('hide', '') if t_key and title and content: t_obj = Topic.get_by_key(t_key) if t_obj: t_obj['title'] = title t_obj['content'] = textilize(content) if hide: t_obj['hide'] = hide else: if t_obj.has_key('hide'): del t_obj['hide'] if Commomkvdb.save(t_key, t_obj): self.redirect('/edit-post?key=%s' % t_key) return self.redirect('/edit-post?key=%s' % t_key)
def post(self): t_key = str(self.get_argument('key','')) title = self.get_argument('title','') content = self.get_argument('content','') hide = self.get_argument('hide','') if t_key and title and content: t_obj = Topic.get_by_key(t_key) if t_obj: t_obj['title'] = title t_obj['content'] = textilize(content) if hide: t_obj['hide'] = hide else: if t_obj.has_key('hide'): del t_obj['hide'] if Commomkvdb.save(t_key, t_obj): self.redirect('/edit-post?key=%s'%t_key) return self.redirect('/edit-post?key=%s'%t_key)
def load_topics(): """Load ratings from source CSV into database.""" print "Importing topics..." # Delete all rows in table, so if we need to run this a second time, # we won't be trying to add duplicate retailers Topic.query.delete() # Read CSV file with open("seed_data/topics.csv") as source_file: example_data = list(csv.reader(source_file)) # skip header row for populating db for list_item in example_data[1:]: topic = Topic(topic_title=list_item[1]) # Add the current retailer to the session db.session.add(topic) # Commit the db.session changes to the database db.session.commit()
def get(self, nodeid, topicid): topic_key = '%s-%s' % (nodeid, topicid) t_obj = Topic.get_by_key_name(topic_key) if not t_obj: self.error(404) self.echo('error.html', { 'page': '404', 'title': "Can't find out this URL", 'h2': 'Oh, my god!', 'msg': 'Something seems to be lost...' }) return if t_obj.cnum: from_id = int(self.request.get('id', '1')) if from_id>1 and from_id%EACH_PAGE_COMMENT_NUM!=1: self.redirect('/t-'+topic_key) return to_id = from_id + EACH_PAGE_COMMENT_NUM - 1 if to_id > t_obj.cnum: to_id = t_obj.cnum c_objs = Comment.get_comments(topic_key, from_id, to_id) else: c_objs = None from_id = to_id = cnum = 0 self.echo('topicdetail.html', { 'title': t_obj.title + ' - ' + t_obj.node.name, 'description':'description', 't_obj': t_obj, 'c_objs': c_objs, 'from_id': from_id, 'to_id': to_id, 'newest_node': Node.get_newest(), 'recent_node': Node.get_recent_node(), 'hot_node': Node.get_hot_node(), 'recent_topic_objs': KeyStrValue.get_topic_key_title('recent-topic-home'), 'reply_topic_objs': KeyStrValue.get_topic_key_title('recent-reply-topic'), }, layout='_layout.html')
def get(self, nodeid='1'): if self.cur_user and self.cur_user.flag>1: n_obj = Node.get_by_id(int(nodeid)) if not n_obj: self.error(404) self.echo('error.html', { 'page': '404', 'title': "Can't find out this URL", 'h2': 'Oh, my god!', 'msg': 'Something seems to be lost...' }) return self.echo('newpost.html', { 'title': "发新帖子", 'errors':[], 'n_obj': n_obj, 't_obj': Topic(), 'newest_node': Node.get_newest(), }, layout='_layout.html') else: self.error(403) self.write('403:forbidden')
def post(self): args = parser.parse_args() try: modified, last_id = Post.create(args) result = dict(success=modified > 0) if modified: result['post'] = Post.find(last_id[0]['last_id']) last_id = last_id[0]['last_id'] topics = args['topics'] topic_obj = [] print(topics) for topic in topics.split(','): topic = topic.strip() r = Topic.get_create_new(topic) if r is None: raise Exception("Unable to create topic") topic_obj.append(r) Post.add_topic(last_id, {'topic_id': r['topic_id']}) result['topics'] = topic_obj return result except Exception as e: return dict(success=False, message=str(e))
def create_poll(): if True: #gets the email stored at the front end #gets the user #choices would be sent as query and number cache_ = request.args.get("cache") if cache_: email = request.args.get("email") name = request.args.get("name") topic = request.args.get("topic") imagename = request.args.get("image") cache.new_entry(email, name, topic, 0, imagename) return jsonify(json_message("ok", "new pending option added")) save = None if request.method == "POST": save = request.form["save"] if save: if "file" not in request.files: return jsonify(json_message("error", "no file selected")) file = request.files["file"] secure_name = secure_filename(file.filename) file.save(os.path.join(vote.config["UPLOAD_FOLDER"], secure_name)) return jsonify(json_message("ok", "new file added")) email = request.args.get("email") user = User.query.filter_by(email=email).first() Topic_ = request.args.get("topic") T = Topic(title=Topic_, count=0, users=user) for choice_ in cache.get_entry(email): T.choices.append( choice(name=choice_[0], count=0, Topic=T, imagename=choice_[3])) ip.new_entry(Topic_, "") user.topics.append(T) db.session.commit() cache.delete_entry(email) return jsonify(json_message("ok", "you craeted a new poll")) else: return jsonify(json_message("error", "not logged in"))
def load_topic_data(): """Load data about the topics from the topic data file""" Data_file = open('seed_data/topic_data.txt') for line in Data_file: line = line.strip() row = line.split('|') topic_title = row[1] zoom = row[2] maxzoom = row[4] minzoom = row[3] band1 = row[5] band2 = row[6] m_date = row[7] main_date = date_treatment(m_date) center_lat = row[8] center_lng = row[9] image = row[10] description = row[11] band3 = row[12] createdby = -111 topic_data = Topic(topic_title=topic_title, zoom=zoom, maxzoom=maxzoom, minzoom=minzoom, band1=band1, band2=band2, main_date=main_date, center_lat=center_lat, center_lng=center_lng, image=image, description=description, band3=band3, createdby=createdby) db.session.add(topic_data) db.session.commit()
def get(self): (forum, siteroot, tmpldir) = forum_siteroot_tmpldir_from_url(self.request.path_info) if not forum or forum.is_disabled: return self.error(HTTP_NOT_FOUND) cached_feed = memcache.get(rss_memcache_key(forum)) if cached_feed is not None: self.response.headers['Content-Type'] = 'text/xml' self.response.out.write(cached_feed) return feed = feedgenerator.Atom1Feed( title = forum.title or forum.url, link = my_hostname() + siteroot + "rss", description = forum.tagline) topics = Topic.gql("WHERE forum = :1 AND is_deleted = False ORDER BY created_on DESC", forum).fetch(25) for topic in topics: title = topic.subject link = my_hostname() + siteroot + "topic?id=" + str(topic.key().id()) first_post = Post.gql("WHERE topic = :1 ORDER BY created_on", topic).get() msg = first_post.message # TODO: a hack: using a full template to format message body. # There must be a way to do it using straight django APIs name = topic.created_by if name: t = Template("<strong>{{ name }}</strong>: {{ msg|striptags|escape|urlize|linebreaksbr }}") else: t = Template("{{ msg|striptags|escape|urlize|linebreaksbr }}") c = Context({"msg": msg, "name" : name}) description = t.render(c) pubdate = topic.created_on feed.add_item(title=title, link=link, description=description, pubdate=pubdate) feedtxt = feed.writeString('utf-8') self.response.headers['Content-Type'] = 'text/xml' self.response.out.write(feedtxt) memcache.add(rss_memcache_key(forum), feedtxt)
def get_render_topic_list(self, list_type='latest'): page = int(self.request.get('page', 1)) user = users.get_current_user() user_key = user.user_id() if user else 'guest' topic_list = memcache.get('topic_list::'+list_type+'::'+user_key) if not topic_list or page != 1: offset = (page - 1) * ITEMS_PER_PAGE topic = Topic.all() if list_type == 'latest': topic.order('-date') elif list_type == 'top': topic.order('-voteup') topic.order('votedown') topic_list = topic.fetch(limit=ITEMS_PER_PAGE, offset=offset) if page == 1: if type == 'top': memcache.set('topic_list::top::'+user_key, topic_list, 1800) else: memcache.set('topic_list::latest::'+user_key, topic_list, 60) return self.get_render('topic_list', {'list': topic_list, 'list_length': len(topic_list), 'page': page+1})
def post(self): user = users.get_current_user() if not user: self.response.out.write('~!@#$%') return if (not self.request.get('sentence')): self.response.out.write( json_output('error', { 'message': 'empty sentence', })) return if (not self.request.get('answer')): self.response.out.write( json_output('error', { 'message': 'empty answer', })) return topic = Topic() topic.author = users.get_current_user() topic.sentence = self.request.get('sentence')[:300] topic.answer = self.request.get('answer') topic.put() memcache.delete('topic_list::latest' + user.user_id()) if self.is_ajax: self.response.out.write( json_output( 'ok', { 'user': topic.author.nickname(), 'sentence': topic.sentence, 'answer': topic.answer, })) else: self.redirect('/')
def main(): design_patterns = Topic('Design Patterns') patterns_intro = Lecture('Intro to Design Patterns') design_patterns.add_module(patterns_intro) composite = Topic('Composite Pattern') design_patterns.add_module(composite) composite_intro = Lecture('Intro to Composite Pattern') composite.add_module(composite_intro) composite_video = Video("Let's compose!") composite.add_module(composite_video) observer = Topic('Iterator Pattern') design_patterns.add_module(observer) observer_intro = Lecture('Intro to Observer Pattern') observer.add_module(observer_intro) design_patterns.display('\t')
def post(self): forum = Forum.from_url(self.request.path_info) is_admin = users.is_current_user_admin() user = users.get_current_user() if not forum or (forum.is_disabled and not is_admin): return self.redirect("/") if not is_admin and not self.settings.check_ip(self.request.remote_addr): return self.response.out.write('Your IP address has been banned') if not is_admin and not self.settings.check_user( user ): return self.redirect( users.create_login_url("/") ) name = self.request.get('name').strip() email = self.request.get('email').strip() subject = self.request.get('subject').strip() message = to_unicode( self.request.get('message') ).strip() homepage = self.request.get('homepage').strip() homepage = "" if homepage == "http://" else homepage remember = bool(self.request.get('remember')) try: topic_id = int( self.request.get('topic_id') or 0 ) if topic_id: topic = db.get(db.Key.from_path('Topic', topic_id)) else: topic = None except ValueError: topic = None if topic and topic.is_locked: return self.redirect( "%stopic?id=%s" % (forum.root(), topic.id) ) # Perform simple validation errors = { 'valid': True } # First post must have a subject if not topic and not subject: errors['valid'] = False errors['subject'] = "Subject required for new topic" if not message: errors['valid'] = False errors['message'] = "Message is required" # sha.new() doesn't accept Unicode strings, so convert to utf8 first sha1_digest = sha.new( message.encode('UTF-8') ).hexdigest() if Post.gql("WHERE sha1_digest = :1 AND topic = :2", sha1_digest, topic).get(): errors['valid'] = False errors['message'] = "This is a duplicate post" if not errors['valid']: return self.template_out("skins/default/post.html", { 'isadmin': is_admin, 'user': user, 'errors': errors, 'forum': forum, 'topic': topic and { 'id': topic_id, 'subject': topic.subject }, 'post': { 'message': message, 'subject': subject }, 'fuser': { 'name': name, 'email': email, 'homepage': homepage, 'remember_me': remember } }) # Get user either by google user id or cookie. Create user objects if don't already exist if user: fuser = FofouUser.gql("WHERE user = :1", user).get() else: fuser = FofouUser.gql("WHERE cookie = :1", self.cookie).get() if not fuser: fuser = FofouUser( user = user or users.User('*****@*****.**'), remember_me = remember, email = email or '*****@*****.**', name = name or 'Anonymous', homepage = homepage, cookie = self.cookie ) else: fuser.remember_me = remember fuser.email = email or '*****@*****.**' fuser.name = name or 'Anonymous' fuser.homepage = homepage if not topic: topic = Topic(forum=forum, subject=subject, created_by=fuser.name) forum.num_topics += 1 else: topic.ncomments += 1 forum.num_posts += 1 topic.put() fuser.put() post = Post( topic = topic, user = fuser, user_ip = self.request.remote_addr, message = message, sha1_digest = sha1_digest, user_name = fuser.name, user_email = fuser.email, user_homepage = homepage ) post.put() forum.put() self.redirect( "%stopic?id=%s" % (forum.root(), topic.id) )
from __future__ import print_function import sys from model import Session, User, Topic session = Session() # Add a user to a topic. t1 = Topic("and now for something completely different") t1.author = User("xyzzy") # Add some topics to a user. u2 = User("plugh") u2.topics = [ Topic("my hovercraft is full of eels"), Topic("he is pining for the fjords"), ] try: session.add(t1) session.add(u2) session.commit() except Exception as e: print("Failed:", e) session.rollback() sys.exit(1)
from __future__ import print_function from model import Session, User, Topic, UsernameTaken session = Session() # Create a user. try: user = User.create(session, "foobar") except UsernameTaken: print("username is already taken") # Get a user by id. user = User.get_by_id(session, 1) print(user) # Create a new topic user = User.get_by_id(session, 1) topic = Topic.create(session, user, "albatross for sale") print(topic) # Upvote a topic user = User.get_by_id(session, 1) topic = Topic.get_by_id(session, 1) topic.upvote(session, user) print(topic)
def post(self, nodeid, topicid): if self.cur_user and self.cur_user.flag>1: author = self.cur_user.name content = self.POST['content'] if content and len(content)<=COMMENT_MAX_S: int_time = int(time()) #check spam mi_obj = MemberInfo.get_or_insert(author) if mi_obj.replyt: t_list = mi_obj.replyt.split(',') if len(t_list) == MEMBER_RECENT_REPLY and (int_time-int(t_list[-1])) < 3600: self.write(u'403:不要回复太频繁了 <a href="/t-%s-%s">请返回</a>' % (nodeid, topicid)) return #check repeat content = textilize(content) #content = safe_encode(content) con_md5 = md5(content.encode('utf-8')).hexdigest() if memcache.get('c_'+con_md5): self.write(u'403:请勿灌水 <a href="/t-%s-%s">请返回</a>' % (nodeid, topicid)) return else: memcache.set('c_'+con_md5, '1', 36000) topic_key = '%s-%s' % (nodeid, topicid) t_obj = Topic.get_by_key_name(topic_key) if not t_obj: self.error(404) self.write('404: not found') return if t_obj.cnum: id_num = t_obj.cnum + 1 else: id_num = 1 c_key = '%s-%d' % (topic_key, id_num) c_obj = Comment(key_name=c_key) c_obj.author = author c_obj.add = int_time c_obj.con = content c_obj.put() if c_obj.is_saved(): #topic commont count +1 t_obj.cnum = id_num t_obj.reply = author t_obj.edit = int_time db.run_in_transaction(obj_runput,t_obj) #memberinfo mi_obj.replyn += 1 if mi_obj.replyk: t_list = mi_obj.replyk.split(',') if topic_key in t_list: t_list.remove(topic_key) t_list.insert(0, topic_key) mi_obj.replyk = ','.join(t_list[:MEMBER_RECENT_REPLY]) else: mi_obj.replyk = topic_key if mi_obj.replyt: t_list = mi_obj.replyt.split(',') t_list.insert(0, str(int_time)) mi_obj.replyt = ','.join(t_list[:MEMBER_RECENT_REPLY]) else: mi_obj.replyt = str(int_time) db.run_in_transaction(obj_runput,mi_obj,['reply_objs:'+author]) #recent reply +key hi_obj = KeyStrValue.get_or_insert('recent-reply-topic') if hi_obj.value: t_list = hi_obj.value.split(',') if topic_key in t_list: t_list.remove(topic_key) t_list.insert(0, topic_key) hi_obj.value = ','.join(t_list[:RECENT_COMMENT_NUM]) db.run_in_transaction(obj_runput,hi_obj,['get_topic_key_title:recent-reply-topic']) else: hi_obj.value = topic_key db.run_in_transaction(obj_runput,hi_obj,['get_topic_key_title:recent-reply-topic']) #all reply counter +1 at_obj = Counter.get_or_insert('all-reply-num', name = 'all-reply-num') at_obj.value += 1 db.run_in_transaction(obj_runput,at_obj) #notifications if t_obj.author != author: mentions = findall_mentions(c_obj.con+' @%s '%t_obj.author, author) else: mentions = findall_mentions(c_obj.con, author) if mentions: deferred.defer(set_mentions, topic_key, ','.join(mentions), _countdown=8, _queue="default") #del cache cache_keys = [] hi_obj = KeyStrValue.get_or_insert('recent-topic-home') if hi_obj.value: if topic_key in hi_obj.value.split(','): cache_keys.append('get_topic_objs:recent-topic-home') if id_num<EACH_PAGE_COMMENT_NUM: cache_keys.append('get_comments:%s:1' % topic_key) else: cache_keys.append('get_comments:%s:%d' % (topic_key, [i for i in range(1,id_num,EACH_PAGE_COMMENT_NUM)][-1])) if cache_keys: memcache.delete_multi(cache_keys) self.redirect('/t-%s#reply%d'%(topic_key,id_num)) return else: self.write('错误: 没有内容 或 内容太长了,请后退返回修改!') else: self.error(403) self.write('403:forbidden')
except lite.Error, e: print "Error %s" % e.args[0] except Exception, e: print "Error %s" % e print "Importing topics in which subject ?" current_subject = select_subject(session) if not current_subject: sys.exit(1) con.row_factory = lite.Row cur = con.cursor() cur.execute("Select did_unit, title from topics") rows = cur.fetchall() for row in rows: topic = Topic(did_unit=row['did_unit'], title=row['title']) topic.subject = current_subject session.add(topic) session.commit() print "Importing question in each topic..." # current_topic = select_topic(session, current_subject) # if not current_topic: # sys.exit(1) cur.execute("Select question, qtype, answers, topic_id from questions") rows = cur.fetchall() for row in rows: qa = Question(row['qtype'], row['question']) qa.topic_id = int(row['topic_id'])
def post(self, nodeid='1'): if self.cur_user and self.cur_user.flag>1: n_obj = Node.get_by_id(int(nodeid)) if not n_obj: self.error(404) self.write('404: not found') return errors = [] author = self.cur_user.name title = self.POST['title'] content = self.POST['content'] if title and content: if len(title)<=TITLE_MAX_S and len(content)<=CONTENT_MAX_S: int_time = int(time()) #check spam mi_obj = MemberInfo.get_or_insert(author) if mi_obj.topict: t_list = mi_obj.topict.split(',') if len(t_list) == MEMBER_RECENT_TOPIC and (int_time-int(t_list[-1])) < 3600: self.write(u'403:不要发帖太频繁了 <a href="/newpost/%s">请返回</a>' % nodeid) return #check repeat content = textilize(content) #content = safe_encode(content) con_md5 = md5(content.encode('utf-8')).hexdigest() if memcache.get('c_'+con_md5): self.write(u'403:请勿灌水 <a href="/newpost/%s">请返回</a>' % nodeid) return else: memcache.set('c_'+con_md5, '1', 36000) if n_obj.count: topic_id = n_obj.count + 1 else: topic_id = 1 topic_key = '%s-%s' % (nodeid, str(topic_id)) t_obj = Topic(key_name=topic_key) t_obj.title = title t_obj.author = author t_obj.add = int_time t_obj.con = textilize(content) #t_obj.con = safe_encode(content) t_obj.put() if t_obj.is_saved(): #node count +1 n_obj.count += 1 db.run_in_transaction(obj_runput,n_obj) #memberinfo mi_obj.topicn += 1 if mi_obj.topick: t_list = mi_obj.topick.split(',') t_list.insert(0, topic_key) mi_obj.topick = ','.join(t_list[:MEMBER_RECENT_TOPIC]) else: mi_obj.topick = topic_key if mi_obj.topict: t_list = mi_obj.topict.split(',') t_list.insert(0, str(int_time)) mi_obj.topict = ','.join(t_list[:MEMBER_RECENT_TOPIC]) else: mi_obj.topict = str(int_time) db.run_in_transaction(obj_runput,mi_obj,['topic_objs:'+author]) #recent in home +key hi_obj = KeyStrValue.get_or_insert('recent-topic-home') if hi_obj.value: t_list = hi_obj.value.split(',') t_list.insert(0, topic_key) hi_obj.value = ','.join(t_list[:RECENT_POST_NUM]) else: hi_obj.value = topic_key db.run_in_transaction(obj_runput,hi_obj,['get_topic_objs:recent-topic-home', 'get_topic_key_title:recent-topic-home']) #all topic counter +1 at_obj = Counter.get_or_insert('all-topic-num', name = 'all-topic-num') at_obj.value += 1 db.run_in_transaction(obj_runput,at_obj) #notifications mentions = findall_mentions(t_obj.con, author) if mentions: deferred.defer(set_mentions, topic_key, ','.join(mentions), _countdown=8, _queue="default") self.redirect('/t-'+topic_key) return else: t_obj = Topic(title = title, con = content) errors.append(u"注意标题和内容的最大字数限制,当前字数:%s %d" % (len(title), len(content))) else: t_obj = Topic(title = title, con = content) errors.append("标题和内容必填") self.echo('newpost.html', { 'title': "发新帖子", 'errors':errors, 'n_obj': n_obj, 't_obj': t_obj, 'newest_node': Node.get_newest(), }, layout='_layout.html') else: self.error(403) self.write('403:forbidden')
def post(self): user = self.user() if not user: return self.redirect('/?error=noUser') (forum, siteroot, tmpldir) = forum_siteroot_tmpldir_from_url(self.request.path_info) if not forum or forum.is_disabled: logging.info('no forum') return self.redirect("/") if self.request.get('Cancel'): logging.info('cancel') return self.redirect(siteroot) self.send_cookie() vals = ['TopicId', 'Subject', 'Message', 'Url'] (topic_id, subject, message, homepage) = req_get_vals(self.request, vals) message = to_unicode(message) homepage = sanitize_homepage(homepage) tvals = { 'role': self.role, 'username': self.username, 'siteroot' : siteroot, 'forum' : forum, "prevSubject" : subject, "prevMessage" : message, "prevUrl" : homepage, "prevTopicId" : topic_id } # validate captcha and other values errclass = None if not message: errclass = "message_class" # first post must have subject if not topic_id and not subject: errclass = "subject_class" # sha.new() doesn't accept Unicode strings, so convert to utf8 first message_utf8 = message.encode('UTF-8') s = sha.new(message_utf8) sha1_digest = s.hexdigest() duppost = Post.gql("WHERE sha1_digest = :1", sha1_digest).get() if duppost: errclass = "message_class" if errclass: tvals[errclass] = "error" tmpl = os.path.join("templates/post.html") return self.template_out(tmpl, tvals) if not topic_id: topic = Topic(forum=forum, subject=subject, created_by=user.nickname) topic.put() else: topic = db.get(db.Key.from_path('Topic', int(topic_id))) #assert forum.key() == topic.forum.key() topic.ncomments += 1 topic.put() user_ip_str = get_remote_ip() p = Post(topic=topic, forum=forum, user=user, user_ip=0, user_ip_str=user_ip_str, message=message, sha1_digest=sha1_digest, user_homepage = homepage) p.put() memcache.delete(rss_memcache_key(forum)) if topic_id: logging.info('topic id') self.redirect(siteroot + "topic?id=" + str(topic_id)) else: logging.info('no topic id') self.redirect(siteroot)
def post(self, nodeid, topicid): author = str(self.get_cookie('username','')) content = self.get_argument('content','') if author and content and len(content)<=COMMENT_MAX_S: int_time = int(time()) #check spam u_comment_time = kv.get('u_comment_time:'+author) if u_comment_time: tlist = u_comment_time.split(',') if len(tlist)== MEMBER_RECENT_TOPIC and (int_time-int(tlist[-1])) < 3600: self.write(u'403:不要回复太频繁了 <a href="/t-%s-%s">请返回</a>' % (nodeid, topicid)) return #check repeat content = textilize(content) #content = safe_encode(content) con_md5 = md5(content.encode('utf-8')).hexdigest() if mc.get('c_'+con_md5): self.write(u'403:请勿灌水 <a href="/t-%s-%s">请返回</a>' % (nodeid, topicid)) return else: mc.set('c_'+con_md5, '1', 36000) ## t_key = 't-%s-%s' % (str(nodeid), str(topicid)) t_obj = Topic.get_by_key(t_key) if t_obj['cnum']: id_num = int(t_obj['cnum']) + 1 else: id_num = 1 c_key = 't-%s-%s-%d' % (str(nodeid), str(topicid), id_num) c_obj = COMMENT_DICT.copy() c_obj['author'] = author c_obj['add'] = int_time c_obj['content'] = content if Commomkvdb.save(c_key, c_obj): #topic commont count +1 t_obj['cnum'] = id_num t_obj['reply'] = author t_obj['edit'] = int_time Commomkvdb.save(t_key, t_obj) #member recent +key #Member.add_key_rencent_comment_topic(author, t_key) rt_obj = kv.get('comment-topic-'+author) if rt_obj: olist = rt_obj.split(',') if t_key in olist: olist.remove(t_key) olist.insert(0, t_key) kv.set('comment-topic-'+author, ','.join(olist[:MEMBER_RECENT_TOPIC])) else: kv.set('comment-topic-'+author, t_key) #recent comment in home +key Commomkvdb.add_key_rencent_topic('recent-comment-topic-home', t_key) #all comment counter +1 Count.key_incr('all-comment-num') #notifications if t_obj['author'] != author: mentions = findall_mentions(c_obj['content']+' @%s '%t_obj['author'], author) else: mentions = findall_mentions(c_obj['content'], author) if mentions: tqueue = TaskQueue('default') tqueue.add(Task('/task/mentions/'+t_key, 'member='+','.join(mentions), delay=5)) #set for check spam #u_comment_time = kv.get('u_comment_time:'+author) if u_comment_time: tlist = u_comment_time.split(',') if str(int_time) not in tlist: tlist.insert(0, str(int_time)) u_comment_time = ','.join(tlist[:MEMBER_RECENT_TOPIC]) kv.set('u_comment_time:'+author, u_comment_time) else: u_comment_time = str(int_time) kv.set('u_comment_time:'+author, u_comment_time) ##set new sr_code cur_user = self.cur_user() code_list = [cur_user['code']] u_topic_time = kv.get('u_topic_time:'+author) if u_topic_time: code_list.append(u_topic_time) if u_comment_time: code_list.append(u_comment_time) self.set_cookie('usercode', md5(''.join(code_list)).hexdigest(), path="/", expires_days = 365 ) #del cache cachekeys = ['get_topic_by_keys:recent-comment-topic-home', 'get_topic_by_keys:comment-topic-'+author, 'get_comment_topic_by_keys:recent-topic-home', 'get_comment_topic_by_keys:recent-comment-topic-home','cur_user:'******'recent-topic-home') if tks and t_key in tks.split(','): cachekeys.append('get_topic_by_keys:recent-topic-home') if id_num<EACH_PAGE_COMMENT_NUM: cachekeys.append('get_comments:%s:1' % t_key) else: cachekeys.append('get_comments:%s:%d' % (t_key, [i for i in range(1,id_num,EACH_PAGE_COMMENT_NUM)][-1])) clear_cache_multi(cachekeys) self.redirect('/'+t_key) return else: self.set_status(403) self.write('错误: 403 (请返回填写内容 或 内容太长了)')
class Crawler(): def __init__(self, loop=None): self.loop = loop or asyncio.get_event_loop() self.root = 'https://frodo.douban.com/api/v2/group/146409/topics?count=100&start={offset}' self.seen_urls = set() self.done = [] self.q = asyncio.Queue(loop=self.loop) self.session = aiohttp.ClientSession(loop=self.loop) self.headers = {'User-Agent': 'api-client/0.1.3 com.douban.frodo/5.20.0 iOS/11.2.5 iPhone8,4 network/wifi'} self.t0 = time.time() self.t1 = None self.count = 0 self.topic = Topic() async def token(self): login_url = 'https://frodo.douban.com/service/auth2/token' payload = { 'client_id': '0ab215a8b1977939201640fa14c66bab', 'grant_type': 'password', 'client_secret': '22b2cf86ccc81009', 'username': USERNAME, 'password': PASSWORD} async with self.session.post(url=login_url, headers=self.headers, data=payload) as r: json_body = await r.json() self.headers['Authorization'] = 'Bearer ' + json_body['access_token'] def add_url(self, url): self.seen_urls.add(url) self.q.put_nowait(url) async def parse_links(self, resp): links = set() if resp.status == 200: print(resp.content_type) if resp.content_type in ('text/html', 'application/xml'): text = await resp.text() urls = set(re.findall(r'''(?i)href=["']([^\s"'<>]+)''', text)) for url in urls: # 标准化url normalized = urllib.parse.urljoin(str(resp.url), url) defragmented, frag = urllib.parse.urldefrag(normalized) if 'topic' in urlparse(defragmented).path.split('/'): links.add(defragmented) print(defragmented) return links # TODO 解析页面内容 async def parse_josn(self, resp): topics = await resp.json() print(topics['count']) print(topics['start']) for topic in topics['topics']: self.topic.update_time = topic['update_time'] self.topic.title = topic['title'] self.topic.url = topic['url'] self.topic.create_time = topic['create_time'] self.topic.comments_count = topic['comments_count'] self.topic.meta.id = topic['id'] self.topic.save() async def fetch(self, url): tries = 0 while tries < 5: try: resp = await self.session.get(url=url, headers=self.headers, allow_redirects=False) break except aiohttp.ClientError as client_error: print(client_error) tries += 1 else: print('超过尝试次数') try: await self.parse_josn(resp) # links = await self.parse_json(resp) # for link in links.difference(self.seen_urls): # self.q.put_nowait(link) # self.seen_urls.update(links) finally: await resp.release() async def work(self): try: while True: url = await self.q.get() assert url in self.seen_urls await self.fetch(url) self.q.task_done() except asyncio.CancelledError: pass async def crawl(self): await self.token() for offset in range(10): self.add_url(self.root.format(offset=offset*100)) workers = [asyncio.Task(self.work(), loop=self.loop) for _ in range(5)] self.t0 = time.time() await self.q.join() self.t1 = time.time() for w in workers: w.cancel()
def post(self, nodeid='1'): n_obj = Node.get_by_key('n-'+str(nodeid)) if not n_obj: self.set_status(404) self.write('404') return errors = [] author = str(self.get_cookie('username','')) title = self.get_argument('title','') content = self.get_argument('content','') t_obj = TOPIC_DICT.copy() if title and content: if len(title)<=TITLE_MAX_S and len(content)<=CONTENT_MAX_S: int_time = int(time()) #check spam u_topic_time = kv.get('u_topic_time:'+author) if u_topic_time: tlist = u_topic_time.split(',') if len(tlist)== MEMBER_RECENT_TOPIC and (int_time-int(tlist[-1])) < 3600: self.write(u'403:不要发帖太频繁了 <a href="/newpost/%s">请返回</a>' % nodeid) return #check repeat content = textilize(content) #content = safe_encode(content) con_md5 = md5(content.encode('utf-8')).hexdigest() if mc.get('c_'+con_md5): self.write(u'403:请勿灌水 <a href="/newpost/%s">请返回</a>' % nodeid) return else: mc.set('c_'+con_md5, '1', 36000) t_obj['title'] = title t_obj['nodeid'] = str(nodeid) t_obj['nodename'] = n_obj['name'] t_obj['author'] = author t_obj['add'] = int_time t_obj['content'] = content if n_obj['count']: topic_id = int(n_obj['count']) + 1 else: topic_id = 1 if Topic.add(topic_id, t_obj): topic_key = 't-%s-%s' % (str(nodeid), str(topic_id)) #node count +1 n_obj['count'] = str(topic_id) Commomkvdb.save('n-'+str(nodeid), n_obj) #member recent +key #Member.add_key_rencent_topic(author, topic_key) rt_obj = kv.get('topic-'+author) if rt_obj: olist = rt_obj.split(',') if topic_key not in olist: olist.insert(0, topic_key) rt_obj = ','.join(olist[:MEMBER_RECENT_TOPIC]) kv.set('topic-'+author, rt_obj) else: rt_obj = topic_key kv.set('topic-'+author, topic_key) #recent in home +key Commomkvdb.add_key_rencent_topic('recent-topic-home', topic_key) #all topic counter +1 Count.key_incr('all-topic-num') #hot node tqueue = TaskQueue('default') tqueue.add(Task('/task/hotnode/%s/%s' % ('n-'+str(nodeid), str(topic_id)), delay=5)) #notifications mentions = findall_mentions(t_obj['content'], author) if mentions: tqueue.add(Task('/task/mentions/'+topic_key, 'member='+','.join(mentions), delay=8)) #set for check spam #u_topic_time = kv.get('u_topic_time:'+author) if u_topic_time: tlist = u_topic_time.split(',') if str(int_time) not in tlist: tlist.insert(0, str(int_time)) u_topic_time = ','.join(tlist[:MEMBER_RECENT_TOPIC]) kv.set('u_topic_time:'+author, u_topic_time) else: u_topic_time = str(int_time) kv.set('u_topic_time:'+author, u_topic_time) ##set new sr_code cur_user = self.cur_user() code_list = [cur_user['code'],u_topic_time] u_comment_time = kv.get('u_comment_time:'+author) if u_comment_time: code_list.append(u_comment_time) self.set_cookie('usercode', md5(''.join(code_list)).hexdigest(), path="/", expires_days = 365 ) #del cache clear_cache_multi(['get_topic_by_keys:recent-topic-home','get_topic_by_keys:topic-' + author, 'get_comment_topic_by_keys:recent-topic-home', 'get_comment_topic_by_keys:recent-comment-topic-home','cur_user:'******'/'+topic_key) return else: errors.append("服务器出现错误,请稍后再试") else: t_obj['title'] = title t_obj['content'] = content errors.append(u"注意标题和内容的最大字数:%s %d" % (len(title), len(content))) else: errors.append("标题和内容必填") self.echo('newpost.html', { 'title': "发新帖子", 'errors':errors, 'n_obj': n_obj, 't_obj': t_obj, }, layout='_layout.html')
import asyncio import aiohttp from config import USERNAME, PASSWORD from model import Topic login_url = 'https://frodo.douban.com/service/auth2/token' group_url = 'https://frodo.douban.com/api/v2/group/146409/topics' headers = { 'User-Agent': 'api-client/0.1.3 com.douban.frodo/5.20.0 iOS/11.2.5 iPhone8,4 network/wifi', } Topic.init() async def login(): #TODO 从配置文件读账号密码 payload = { 'client_id': '0ab215a8b1977939201640fa14c66bab', 'grant_type': 'password', 'client_secret': '22b2cf86ccc81009', 'username': USERNAME, 'password': PASSWORD } async with aiohttp.ClientSession() as session: #拿调用api的token async with session.post(url=login_url, headers=headers, data=payload) as r: json_body = await r.json()