def tryParseUrl(self, url: str) -> Optional[FicId]: mapPrefixes = ['http://www.', 'http://', 'https://www.'] hasPrefix = True while hasPrefix: hasPrefix = False for pref in mapPrefixes: if url.startswith(pref): hasPrefix = True url = 'https://' + url[len(pref):] endsToStrip = [ '#main', '#work_endnotes', '#bookmark-form', '?view_adult=true', '?view_full_work=true', '?viewfullwork=true', '?show_comments=true', ] for send in endsToStrip: if url.endswith(send): url = url[:-len(send)] if url.find('#') >= 0: url = url[:url.find('#')] if url.find('?') >= 0: url = url[:url.find('?')] # TODO: this should probably return a FicId pointing to this chapter and # not just this fic in general... if url.find('/chapters/') >= 0 and url.find('/works/') < 0: meta = scrape.softScrapeWithMeta(url, delay=10) if meta is None or meta['raw'] is None or meta['status'] != 200: raise Exception('unable to lookup chapter: {}'.format(url)) from bs4 import BeautifulSoup # type: ignore soup = BeautifulSoup(meta['raw'], 'html5lib') for a in soup.find_all('a'): if a.get_text() == 'Entire Work': return self.tryParseUrl(self.baseUrl + a.get('href')[len('/works/'):]) else: raise Exception('unable to lookup chapters entire work: {}'.format(url)) if url.startswith(self.collectionUrl) and url.find('/works/') != -1: url = self.baseUrl + url[url.find('/works/') + len('/works/'):] if not url.startswith(self.baseUrl): return None pieces = url[len(self.baseUrl):].split('/') lid = pieces[0] if len(lid) < 1 or not lid.isnumeric(): return None ficId = FicId(FicType.ao3, lid) fic = Fic.tryLoad(ficId) if fic is None: return ficId if len(pieces) >= 3 and pieces[1] == 'chapters' and pieces[2].isnumeric(): localChapterId = pieces[2] mchaps = FicChapter.select( { 'ficId': fic.id, 'localChapterId': localChapterId } ) if len(mchaps) == 1: ficId.chapterId = mchaps[0].chapterId ficId.ambiguous = False return ficId
def __doRefilter(self, force: bool) -> None: if len(self.filter) < 1: return ficId = FicId.tryParse(self.filter) if ficId is not None: self.idx = 0 self.list = [] if ficId.ambiguous == False: fic = Fic.tryLoad(ficId) if fic is None: fic = Fic.load(ficId) self.fics = Fic.list() self.list = [fic] else: fic = Fic.tryLoad(ficId) if fic is not None: self.list = [fic] return plain: List[str] = [] tags: List[str] = [] tagStarts = ['is:', 'i:', ':'] for w in self.filter.split(): isTag = False for tagStart in tagStarts: if w.startswith(tagStart): tags += [w[len(tagStart):]] isTag = True break if not isTag: plain += [w] # TODO: simplify fcmp tags favRel = None ratRel = None isNew = None isComplete = None authorRel = None fandomRel = None descRel = None titleRel = None for tag in tags: if len(tag) == 0: continue arg: str = '' rel: Optional[str] = None for prel in ['=', '<', '>', '~', '.']: if tag.find(prel) != -1: ps = tag.split(prel) if len(ps) != 2: continue tag = ps[0] rel = prel arg = ps[1] break if 'favorite'.startswith(tag): if rel is not None and arg.isnumeric(): favRel = (rel, int(arg)) else: favRel = ('>', 0) elif 'rated'.startswith(tag): if rel is not None and arg.isnumeric(): ratRel = (rel, int(arg)) else: ratRel = ('>', 0) elif 'new'.startswith(tag): isNew = ('=', 'new') elif 'author'.startswith(tag): if rel is not None: authorRel = (rel, arg) elif 'fandom'.startswith(tag): if rel is not None: fandomRel = (rel, arg) elif 'complete'.startswith(tag): isComplete = ('=', 'complete') elif 'description'.startswith(tag): if rel is not None: descRel = (rel, arg) elif 'title'.startswith(tag): if rel is not None: titleRel = (rel, arg) self.pushMessage('f:{}, r:{}, n:{}, c:{}, a:{}, f2:{} p:{}'.format( favRel, ratRel, isNew, isComplete, authorRel, fandomRel, plain)) pfilter = ' '.join(plain).lower() nfics: List[Fic] = [] completelyRefilter = (force or (self.filter[-1] == ' ' or self.filter[-1] == ':')) # TODO FIXME bleh userFics = {uf.ficId: uf for uf in UserFic.select({'userId': 1})} for fic in (self.fics if completelyRefilter else self.list): if fic.id not in userFics: userFics[fic.id] = UserFic.default((1, fic.id)) userFic = userFics[fic.id] if favRel is not None or ratRel is not None or isNew: if favRel is not None: if not self.fcmp(favRel[0], userFic.isFavorite, favRel[1]): continue if ratRel is not None: if not self.fcmp(ratRel[0], userFic.rating or -1, ratRel[1]): continue if isNew is not None: if userFic.lastChapterViewed != 0: continue if descRel is not None: if not self.fcmp(descRel[0], fic.description or '', descRel[1]): continue if titleRel is not None: if not self.fcmp(titleRel[0], fic.title or '', titleRel[1]): continue if isComplete is not None and fic.ficStatus != FicStatus.complete: continue if authorRel is not None: if not self.fcmp(authorRel[0], fic.getAuthorName(), authorRel[1]): continue if fandomRel is not None: ficFandoms = [fandom.name for fandom in fic.fandoms()] matchesFandom = False for fandom in ficFandoms: if self.fcmp(fandomRel[0], fandom, fandomRel[1]): matchesFandom = True break if not matchesFandom: continue ftext = ( f'{fic.localId} {fic.title} {fic.getAuthorName()} {fic.id}'. lower()) if util.subsequenceMatch(ftext, pfilter): nfics += [fic] self.list = nfics