def _create_model_object(self): random_url = self.faker.uri() random_title = self.faker.sentence() url = Url.find_or_create(self.db.session, random_url, random_title) if self._exists_in_db(url): return self._create_model_object() return url
def find_or_create( cls, session, user, _origin: str, _origin_lang: str, _translation: str, _translation_lang: str, _context: str, _url: str, _url_title: str, article_id: int, ): """ if the bookmark does not exist, it creates it and returns it if it exists, it ** updates the translation** and returns the bookmark object :param _origin: :param _context: :param _url: :return: """ origin_lang = Language.find_or_create(_origin_lang) translation_lang = Language.find_or_create(_translation_lang) origin = UserWord.find_or_create(session, _origin, origin_lang) article = Article.query.filter_by(id=article_id).one() url = Url.find_or_create(session, article.url.as_string(), _url_title) context = Text.find_or_create(session, _context, origin_lang, url, article) translation = UserWord.find_or_create(session, _translation, translation_lang) now = datetime.now() try: # try to find this bookmark bookmark = Bookmark.find_by_user_word_and_text(user, origin, context) # update the translation bookmark.translation = translation except sqlalchemy.orm.exc.NoResultFound as e: bookmark = cls(origin, translation, user, context, now) except Exception as e: raise e session.add(bookmark) session.commit() return bookmark
def find_or_create(cls, session, _url: str, language=None, sleep_a_bit=False): """ If not found, download and extract all the required info for this article. :param url: :return: """ from zeeguu_core.model import Url, Article, Language import newspaper url = Url.extract_canonical_url(_url) try: found = cls.find(url) if found: return found art = newspaper.Article(url=url) art.download() art.parse() if art.text == '': raise Exception("Newspaper got empty article from: " + url) if sleep_a_bit: import time from random import randint print("GOT: " + url) sleep_time = randint(3, 33) print( f"sleeping for {sleep_time}s... so we don't annoy our friendly servers" ) time.sleep(sleep_time) if not language: if art.meta_lang == '': art.meta_lang = detect(art.text) zeeguu_core.log(f"langdetect: {art.meta_lang} for {url}") language = Language.find_or_create(art.meta_lang) # Create new article and save it to DB url_object = Url.find_or_create(session, url) new_article = Article( url_object, art.title, ', '.join(art.authors), art.text[ 0: 32000], # any article longer than this will be truncated... art.summary, None, None, language) session.add(new_article) session.commit() return new_article except sqlalchemy.exc.IntegrityError or sqlalchemy.exc.DatabaseError: for i in range(10): try: session.rollback() u = cls.find(url) print("Found article by url after recovering from race") return u except: print("Exception of second degree in article..." + str(i)) time.sleep(0.3) continue break