def __init__(self): """load basic members using memcache""" logging.info("initialized") self.posts_tags_db=[] self.catdict={} self.posts_tags_dict={} self.posts=memcache.get(KEY) if self.posts is None: logging.info('cache is empty creating index') self.posts = BlogPost.all().order('-timestamp') createIndex(self.posts) memcache.add(KEY,self.posts) if isinstance(self.posts,list):self.nofposts=len(self.posts)-2 else:self.nofposts=self.posts.count()-2 self.tags=memcache.get(TAG) if self.tags is None: self.tags = Tag.all() memcache.add(TAG,self.tags) self.categories=memcache.get(CATEGORY) if self.categories is None: self.categories= Category.all() memcache.add(CATEGORY,self.categories) for post in self.posts: logging.info(['posts',post.title]) self.posts_tags_db.extend(post.tags) tags=[] for key in post.tags:tags.append(db.get(key).tag) self.posts_tags_dict[post.key()]=tags self.catdict[post.category.key()]=post.category.category logging.info(['catdict',self.catdict]) self.tagnames=list(chain.from_iterable(self.posts_tags_dict.values()))
def refetch(self): self.posts =BlogPost.all().order('-timestamp') memcache.add(KEY,self.posts) self.tags = Tag.all() memcache.add(TAG,self.tags) self.categories= Category.all() memcache.add(CATEGORY,self.categories) createIndex(self.posts)
#! /usr/bin/python2 from search import indexListing from textbookExceptions import UnIndexable from mcmaster.classes import allCourses from search import indexListing, createIndex from itertools import imap try: print "Trying to create the index if it does not exist already" createIndex("oersearch") except Exception as e: print e print "Downloading course info" for c in allCourses(): try: print indexListing(c) except UnIndexable as e: print e
def update(self): """updates a post""" #for catobj in self.categories: # logging.info([catobj.key(),catobj.category,catkeys]) # if catobj.category==self.postcatkey:#AN OLD CATEGORY # catkey=catobj.key() # elif catobj.key() not in self.catkeys:#not used category # catobj.delete() # else: # logging.info(catobj.key().id()) #newcatobj=Category() #newcatobj.category=category #newcatobj.put() #catkey=newcatobj.key() post_tagsdb_values=[] post_tagsdb_keys=[] existingTags=[] existingTagskeys=[] tagsleft=[] #find the existing tags of the post for tagkey in self.posts_tags_db: if tagkey not in self.post_tags_keys: try: tagsleft.append(Tag.get_by_id(tagkey.id()).tag) except AttributeError:#ops a post without a tag continue existingTagskeys.append(tagkey) existingTags.append(db.get(tagkey).tag) #existing Tags for tagkey in self.post_tags_keys:post_tagsdb_values.append(db.get(tagkey).tag)#previous Tags of the post # logging.info([self.posttags,type(self.posttags),type(post_tagsdb_values),post_tagsdb_values]) unchangedtags=[] returnedTags=[] # logging.info(['posttags',self.posttags,post_tagsdb_values]) if post_tagsdb_values:#post does have tags logging.info(post_tagsdb_values) unchangedtags=set(self.posttags) & set( post_tagsdb_values)#changes tags added or removed newtags=set(self.posttags) ^ unchangedtags#new tags for this post oldtags=list(set(post_tagsdb_values)^unchangedtags) logging.info(["new",newtags,"old",oldtags,"unchanged",unchangedtags,list(unchangedtags)]) if list(unchangedtags): unchangedtags=list(unchangedtags) for tag in unchangedtags: tagid=db.get(existingTagskeys[existingTags.index(tag)]).key().id() returnedTags.append({"tag":tag,"tagid":tagid}) else:unchangedtags=[] i=0 logging.info(['Tags from other posts',existingTags]) for tag in oldtags:#tags to be removed tag_key= existingTagskeys[existingTags.index(tag)] if tag not in tagsleft:#delete not used tags tagobj=db.get(tag_key) logging.info(["deleting",tag,tagobj]) tagobj.delete() pos=post_tagsdb_values.index(tag) self.obj.tags.pop(pos-i) i+=1 elif self.posttags:#new tags do exist logging.info(self.posttags) newtags=set(self.posttags)#does not have tags else:newtags=[] if newtags: for tag in list(newtags):#add new tag and update Post logging.info(tag) if tag not in existingTags: tagobj=Tag() tagobj.tag=tag tagid=tagobj.put().id() returnedTags.append({"tag":tag,"tagid":tagid}) else: tag_key= existingTagskeys[existingTags.index(tag)] tagobj=Tag.get(tag_key) returnedTags.append({"tag":tagobj.tag,"tagid":tagobj.key().id()}) self.obj.tags.append(tagobj.key()) if isinstance(self.postcategory,list):self.postcategory=self.postcategory[0] logging.info([self.catdict.values()]) self.obj.title=self.title self.obj.body=self.body self.obj.category=self.catdict.keys()[self.catdict.values().index(self.postcategory)] self.obj.updated=datetime.now() self.obj.put() createIndex([self.obj]) tags=[] [tags.append({"tag":db.get(key).tag,"tagid":db.get(key).key().id()}) for key in self.obj.tags] data=[] updated=str(self.obj.updated.day)+" "+str(months[self.obj.updated.month])+" "+str(self.obj.updated.year) dateposted=str(self.obj.timestamp.day)+" "+str(months[self.obj.timestamp.month])+" "+str(self.obj.timestamp.year) data.append({"title":self.obj.title,"body":self.obj.body,"category":db.get(self.obj.category.key()).category, "catid": db.get(self.obj.category.key()).key().id(),"id":str(self.obj.key().id()),\ "tags":tags,"date":dateposted,"updated":updated}) self.deleteMemcache(self) self.refetch(self) return(data,returnedTags)
def main(): #create/open whoosh index and schema createIndex("fishbase-data2.csv") #start the server app.run(debug=True)