def do1(): mpost = MPost() recs = mpost.query_all() for rec in recs: text2 = html2text.html2text(tornado.escape.xhtml_unescape( rec.cnt_html)) print(text2) writer.add_document(title=rec.title, type='<span style="color:green;">[文档]</span>', link='/post/{0}.html'.format(rec.uid), content=text2)
def do1(): mpost = MPost() recs = mpost.query_all() for rec in recs: text2 = html2text.html2text(tornado.escape.xhtml_unescape(rec.cnt_html)) print(text2) writer.add_document( title=rec.title, type='<span style="color:green;">[文档]</span>', link='/post/{0}.html'.format(rec.uid), content=text2 )
def build_whoosh_database(): analyzer = ChineseAnalyzer() schema = Schema(title=TEXT(stored=True, analyzer=analyzer), type=TEXT(stored=True), link=ID(stored=True), content=TEXT(stored=True, analyzer=analyzer)) ix = create_in(whoosh_database, schema) writer = ix.writer() mpost = MPost() recs = mpost.query_all() for rec in recs: text2 = html2text.html2text(tornado.escape.xhtml_unescape(rec.cnt_html)) print(text2) writer.add_document( title=rec.title, type='<span style="color:blue;">[文档]</span>', link='/post/{0}.html'.format(rec.uid), content=text2 ) writer.commit()
def build_whoosh_database(): analyzer = ChineseAnalyzer() schema = Schema(title=TEXT(stored=True, analyzer=analyzer), type=TEXT(stored=True), link=ID(stored=True), content=TEXT(stored=True, analyzer=analyzer)) ix = create_in(whoosh_database, schema) writer = ix.writer() mpost = MPost() recs = mpost.query_all() for rec in recs: text2 = html2text.html2text(tornado.escape.xhtml_unescape( rec.cnt_html)) print(text2) writer.add_document(title=rec.title, type='<span style="color:blue;">[文档]</span>', link='/post/{0}.html'.format(rec.uid), content=text2) writer.commit()
sys.path.append('/opt/torlite/yunsuan') from torlite.model.mpost import MPost from torlite.model.mpage import MPage from torlite.model.mcatalog import MCatalog from maplet.model.app2catalog_model import MApp2Catalog from torlite.model.mpost2catalog import MPost2Catalog from maplet.model.equation_model import MApp if __name__ == '__main__': uu = MApp() tt = uu.get_all() vv = MPost() ss = vv.query_all() ab = MCatalog() ba = ab.query_all() cd = MPage() dc = cd.query_all() with open('site_map_haosou.txt', 'w') as fo: fo.write(raw_text) for x in tt: # print(x.uid) fo.write('http://www.yunsuan.org/app/{0}\n'.format(x.uid)) for y in ss: fo.write('http://www.yunsuan.org/post/{0}.html\n'.format(y.uid))
class PostHandler(BaseHandler): def initialize(self): self.init() self.muser = MUser() self.mpost = MPost() self.mcat = MCatalog() self.cats = self.mcat.query_all() self.mspec = SpesubModel() self.specs = self.mspec.get_all() self.mpost_hist = MPostHist() self.mpost2catalog = MPost2Catalog() self.mpost2reply = MPost2Reply() self.mapp2tag = MPost2Label() self.mrel = MRelation() self.tmpl_router = 'post' if self.get_current_user(): self.userinfo = self.muser.get_by_id(self.get_current_user()) else: self.userinfo = None def get(self, url_str=''): if url_str == '': self.recent() url_arr = url_str.split('/') if len(url_arr) == 1 and url_str.endswith('.html'): self.wiki(url_str.split('.')[0]) elif url_str == 'add_document': self.to_add_document() elif url_str == 'recent': self.recent() elif url_str == 'refresh': self.refresh() elif (url_arr[0] == 'modify'): self.to_modify(url_arr[1]) elif url_arr[0] == 'delete': self.delete(url_arr[1]) else: kwd = { 'info': '页面未找到', } self.render('html/404.html', kwd=kwd) def post(self, url_str=''): if url_str == '': return url_arr = url_str.split('/') if len(url_arr) == 1 and url_str.endswith('.html'): self.add_post() if url_arr[0] == 'modify': self.update(url_arr[1]) elif url_str == 'add_document': self.user_add_post() elif url_arr[0] == 'add_document': self.user_add_post() else: self.redirect('/html/404.html') def recent(self, with_catalog=True, with_date=True): kwd = { 'pager': '', 'unescape': tornado.escape.xhtml_unescape, 'title': '最近文档', 'with_catalog': with_catalog, 'with_date': with_date, } self.render('{0}/{1}/post_list.html'.format(self.tmpl_name, self.tmpl_router), kwd=kwd, view=self.mpost.query_recent(), view_all=self.mpost.query_all(), format_date=tools.format_date, userinfo=self.userinfo, ) def refresh(self): kwd = { 'pager': '', 'title': '最近文档', } self.render('{0}/{1}/post_list.html'.format(self.tmpl_name, self.tmpl_router), kwd=kwd, userinfo=self.userinfo, view=self.mpost.query_dated(10), format_date=tools.format_date, unescape=tornado.escape.xhtml_unescape, ) def get_random(self): return self.mpost.query_random() def wiki(self, uid): dbdate = self.mpost.get_by_id(uid) if dbdate: self.mpost.update_view_count_by_uid(dbdate.uid) self.viewit(uid) else: self.to_add(uid) def to_add_document(self, ): kwd = { 'pager': '', 'cats': self.cats, 'specs': self.specs, 'uid': '', } self.render('{0}/{1}/post_add.html'.format(self.tmpl_name, self.tmpl_router), topmenu='', kwd=kwd, tag_infos=self.mcat.query_all(), userinfo=self.userinfo, ) @tornado.web.authenticated def to_add(self, uid): kwd = { 'cats': self.cats, 'specs': self.specs, 'uid': uid, 'pager': '', } self.render('{0}/{1}/post_add.html'.format(self.tmpl_name, self.tmpl_router), kwd=kwd, tag_infos=self.mcat.query_all(), ) @tornado.web.authenticated def update(self, uid): raw_data = self.mpost.get_by_id(uid) if self.userinfo.privilege[2] == '1' or raw_data.user_name == self.get_current_user(): pass else: return False post_data = {} for key in self.request.arguments: post_data[key] = self.get_arguments(key) post_data['user_name'] = self.get_current_user() is_update_time = True if post_data['is_update_time'][0] == '1' else False self.update_tag(uid) self.mpost.update(uid, post_data, update_time=is_update_time) self.update_catalog(uid) self.mpost_hist.insert_data(raw_data) self.redirect('/post/{0}.html'.format(uid)) def update_tag(self, signature): if self.userinfo.privilege[4] == '1': pass else: return False post_data = {} for key in self.request.arguments: post_data[key] = self.get_arguments(key) if 'tags' in post_data: pass else: return False current_tag_infos = self.mapp2tag.get_by_id(signature) tags_arr = [x.strip() for x in post_data['tags'][0].split(',')] for tag_name in tags_arr: if tag_name == '': pass else: self.mapp2tag.add_record(signature, tag_name, 1) for cur_info in current_tag_infos: if cur_info.tag.name in tags_arr: pass else: self.mapp2tag.remove_relation(signature, cur_info.tag) @tornado.web.authenticated def update_catalog(self, uid): raw_data = self.mpost.get_by_id(uid) if self.userinfo.privilege[2] == '1' or raw_data.user_name == self.get_current_user(): pass else: return False post_data = {} for key in self.request.arguments: post_data[key] = self.get_arguments(key) current_infos = self.mpost2catalog.query_by_id(uid) new_tag_arr = [] for key in ['tag1', 'tag2', 'tag3', 'tag4', 'tag5']: if key in post_data: pass else: continue if post_data[key][0] == '': pass else: new_tag_arr.append(int(post_data[key][0])) self.mpost2catalog.add_record(uid, post_data[key][0], int(key[-1])) for cur_info in current_infos: if cur_info.catalog.uid not in new_tag_arr: self.mpost2catalog.remove_relation(uid, cur_info.catalog) # .delete(cur_info.uid) @tornado.web.authenticated def to_modify(self, id_rec): a = self.mpost.get_by_id(id_rec) # 用户具有管理权限, # 或 # 文章是用户自己发布的。 if self.userinfo.privilege[2] == '1' or a.user_name == self.get_current_user(): pass else: return False id_spec = a.id_spec kwd = { 'pager': '', 'cats': self.cats, 'specs': self.specs, 'id_spec': id_spec, } self.render('{0}/{1}/post_edit.html'.format(self.tmpl_name, self.tmpl_router), kwd=kwd, unescape=tornado.escape.xhtml_unescape, tag_infos=self.mcat.query_all(), app2label_info=self.mapp2tag.get_by_id(id_rec), app2tag_info=self.mpost2catalog.query_by_id(id_rec), dbrec=a, userinfo=self.userinfo, ) def get_cat_str(self, cats): cat_arr = cats.split(',') out_str = '' for xx in self.cats: if str(xx.uid) in cat_arr: tmp_str = '''<li><a href="/category/{0}" style="margin:10px auto;"> {1} </a></li> '''.format(xx.slug, tornado.escape.xhtml_escape(xx.name)) out_str += tmp_str return (out_str) def get_cat_name(self, id_cat): for x in self.cats: if x['id_cat'] == id_cat: return (x['name']) def viewit(self, post_id): last_post_id = self.get_secure_cookie('last_post_uid') if last_post_id: last_post_id = last_post_id.decode('utf-8') self.set_secure_cookie('last_post_uid', post_id) if last_post_id and self.mpost.get_by_id(last_post_id): self.add_relation(last_post_id, post_id) cats = self.mpost2catalog.query_catalog(post_id) replys = self.mpost2reply.get_by_id(post_id) tag_info = self.mapp2tag.get_by_id(post_id) rec = self.mpost.get_by_id(post_id) if not rec: kwd = { 'info': '您要找的云算应用不存在。', } self.render('html/404.html', kwd=kwd) return False if cats.count() == 0: cat_id = '' else: cat_id = cats.get().catalog kwd = { 'pager': '', 'editable': self.editable(), 'cat_id': cat_id } self.render('{0}/{1}/post_view.html'.format(self.tmpl_name, self.tmpl_router), view=rec, unescape=tornado.escape.xhtml_unescape, kwd=kwd, userinfo=self.userinfo, tag_info=tag_info, relations=self.mrel.get_app_relations(rec.uid), replys=replys, ) def add_relation(self, f_uid, t_uid): if False == self.mpost.get_by_id(t_uid): return False if f_uid == t_uid: ''' 关联其本身 ''' return False # 双向关联,但权重不一样. self.mrel.add_relation(f_uid, t_uid, 2) self.mrel.add_relation(t_uid, f_uid, 1) return True @tornado.web.authenticated def add_post(self): if self.userinfo.privilege[1] == '1': pass else: return False post_data = {} for key in self.request.arguments: post_data[key] = self.get_arguments(key) post_data['user_name'] = self.get_current_user() id_post = post_data['uid'][0] cur_post_rec = self.mpost.get_by_id(id_post) if cur_post_rec is None: uid = self.mpost.insert_data(id_post, post_data) self.update_tag(uid) self.update_catalog(uid) self.redirect('/post/{0}.html'.format(id_post)) @tornado.web.authenticated def user_add_post(self): if self.userinfo.privilege[1] == '1': pass else: return False post_data = {} print(post_data) for key in self.request.arguments: post_data[key] = self.get_arguments(key) post_data['user_name'] = self.get_current_user() cur_uid = tools.get_uu5d() while self.mpost.get_by_id(cur_uid): cur_uid = tools.get_uu5d() uid = self.mpost.insert_data(cur_uid, post_data) self.update_tag(uid) self.update_catalog(uid) self.redirect('/post/{0}.html'.format(cur_uid)) @tornado.web.authenticated def delete(self, del_id): is_deleted = self.mpost.delete(del_id) if self.tmpl_router == "post": if is_deleted: self.redirect('/post/recent') else: return False else: if is_deleted: output = { 'del_info ': 1, } else: output = { 'del_info ': 0, } return json.dump(output, self)
return "".join(result) from whoosh.index import create_in,open_dir from whoosh.fields import * from whoosh.qparser import QueryParser from jieba.analyse import ChineseAnalyzer sys.path.append('/opt/torlite/yunsuan') from torlite.model.mpost import MPost from torapp.model.app_model import MApp mpost = MPost() recs = mpost.query_all() # title, cnt_html analyzer = ChineseAnalyzer() schema = Schema(title=TEXT(stored=True, analyzer = analyzer), type=TEXT(stored=True), link=ID(stored=True), content=TEXT(stored=True, analyzer=analyzer)) if not os.path.exists("lib/whoosh"): os.mkdir("lib/whoosh") ix = create_in("lib/whoosh", schema) # for create new index # ix = open_dir("tmp") # for read only writer = ix.writer() for rec in recs: # print(rec.title, rec.uid, rec.cnt_html)
sys.path.append('/opt/torlite/yunsuan') from torlite.model.mpost import MPost from torlite.model.mpage import MPage from torlite.model.mcatalog import MCatalog from maplet.model.app2catalog_model import MApp2Catalog from torlite.model.mpost2catalog import MPost2Catalog from maplet.model.equation_model import MApp if __name__ == '__main__': uu = MApp() tt = uu.get_all() vv = MPost() ss = vv.query_all() ab = MCatalog() ba = ab.query_all() cd = MPage() dc = cd.query_all() with open('site_map_haosou.txt', 'w') as fo: fo.write(raw_text) for x in tt: # print(x.uid) fo.write('http://www.yunsuan.org/app/{0}\n'.format(x.uid)) for y in ss: fo.write('http://www.yunsuan.org/post/{0}.html\n'.format(y.uid))