def saveMapReduce(namefic): """ Converted the pdf file in text Do the mapper and reduce in the text Get the references cited in the article Save data Author, Article, ArtCitedBib, MapReduce, Master :param namefic : the name file """ # save_pdf(namefic) fic = convert_pdf_to_txt(namefic) dataDict = mapper(fic) dataDict = reducer(dataDict) lines = re.split(r"\n", fic) authorStr = re.sub(r"[^a-zA-Z\s]", " ", lines[4]) author = Author(name=authorStr) author.put() titre = re.sub(r"[^a-zA-Z\s]", " ", lines[0] + lines[1]) titre = titre.strip() article = Article(name=titre, fileName=namefic) article.put() getReferences(fic, article) artiAuth = ArtiAuth(keyAuthor=author, keyArticle=article) artiAuth.put() for cle in dataDict.keys(): mapReduce = MapReduce(keyWord=cle, keyArticle=article, count=dataDict[cle]) mapReduce.put() checkMaster = Master.all() checkMaster.filter("keyWord =", cle) if checkMaster.count() > 0: master = checkMaster.get() master.count = master.count + dataDict[cle] else: master = Master(keyWord=cle, count=dataDict[cle]) master.put()
def deleteData(): """ Delete all the data """ masters = Master.all() for master in masters: Master.delete(master) mapReduces = MapReduce.all() for mapReduce in mapReduces: MapReduce.delete(mapReduce) articles = Article.all() for article in articles: Article.delete(article) authors = Author.all() for author in authors: Author.delete(author) artCitedBibs = ArtCitedBib.all() for artCitedBib in artCitedBibs: ArtCitedBib.delete(artCitedBib)