def tryParseUrl(self, url: str) -> Optional[FicId]:
mapPrefixes = ['http://www.', 'http://', 'https://www.']
hasPrefix = True
while hasPrefix:
hasPrefix = False
for pref in mapPrefixes:
if url.startswith(pref):
hasPrefix = True
url = 'https://' + url[len(pref):]
endsToStrip = [
'#main',
'#work_endnotes',
'#bookmark-form',
'?view_adult=true',
'?view_full_work=true',
'?viewfullwork=true',
'?show_comments=true',
]
for send in endsToStrip:
if url.endswith(send):
url = url[:-len(send)]
if url.find('#') >= 0:
url = url[:url.find('#')]
if url.find('?') >= 0:
url = url[:url.find('?')]
# TODO: this should probably return a FicId pointing to this chapter and
# not just this fic in general...
if url.find('/chapters/') >= 0 and url.find('/works/') < 0:
meta = scrape.softScrapeWithMeta(url, delay=10)
if meta is None or meta['raw'] is None or meta['status'] != 200:
raise Exception('unable to lookup chapter: {}'.format(url))
from bs4 import BeautifulSoup # type: ignore
soup = BeautifulSoup(meta['raw'], 'html5lib')
for a in soup.find_all('a'):
if a.get_text() == 'Entire Work':
return self.tryParseUrl(self.baseUrl + a.get('href')[len('/works/'):])
else:
raise Exception('unable to lookup chapters entire work: {}'.format(url))
if url.startswith(self.collectionUrl) and url.find('/works/') != -1:
url = self.baseUrl + url[url.find('/works/') + len('/works/'):]
if not url.startswith(self.baseUrl):
return None
pieces = url[len(self.baseUrl):].split('/')
lid = pieces[0]
if len(lid) < 1 or not lid.isnumeric():
return None
ficId = FicId(FicType.ao3, lid)
fic = Fic.tryLoad(ficId)
if fic is None:
return ficId
if len(pieces) >= 3 and pieces[1] == 'chapters' and pieces[2].isnumeric():
localChapterId = pieces[2]
mchaps = FicChapter.select(
{
'ficId': fic.id,
'localChapterId': localChapterId
}
)
if len(mchaps) == 1:
ficId.chapterId = mchaps[0].chapterId
ficId.ambiguous = False
return ficId
def __doRefilter(self, force: bool) -> None:
if len(self.filter) < 1:
return
ficId = FicId.tryParse(self.filter)
if ficId is not None:
self.idx = 0
self.list = []
if ficId.ambiguous == False:
fic = Fic.tryLoad(ficId)
if fic is None:
fic = Fic.load(ficId)
self.fics = Fic.list()
self.list = [fic]
else:
fic = Fic.tryLoad(ficId)
if fic is not None:
self.list = [fic]
return
plain: List[str] = []
tags: List[str] = []
tagStarts = ['is:', 'i:', ':']
for w in self.filter.split():
isTag = False
for tagStart in tagStarts:
if w.startswith(tagStart):
tags += [w[len(tagStart):]]
isTag = True
break
if not isTag:
plain += [w]
# TODO: simplify fcmp tags
favRel = None
ratRel = None
isNew = None
isComplete = None
authorRel = None
fandomRel = None
descRel = None
titleRel = None
for tag in tags:
if len(tag) == 0:
continue
arg: str = ''
rel: Optional[str] = None
for prel in ['=', '<', '>', '~', '.']:
if tag.find(prel) != -1:
ps = tag.split(prel)
if len(ps) != 2:
continue
tag = ps[0]
rel = prel
arg = ps[1]
break
if 'favorite'.startswith(tag):
if rel is not None and arg.isnumeric():
favRel = (rel, int(arg))
else:
favRel = ('>', 0)
elif 'rated'.startswith(tag):
if rel is not None and arg.isnumeric():
ratRel = (rel, int(arg))
else:
ratRel = ('>', 0)
elif 'new'.startswith(tag):
isNew = ('=', 'new')
elif 'author'.startswith(tag):
if rel is not None:
authorRel = (rel, arg)
elif 'fandom'.startswith(tag):
if rel is not None:
fandomRel = (rel, arg)
elif 'complete'.startswith(tag):
isComplete = ('=', 'complete')
elif 'description'.startswith(tag):
if rel is not None:
descRel = (rel, arg)
elif 'title'.startswith(tag):
if rel is not None:
titleRel = (rel, arg)
self.pushMessage('f:{}, r:{}, n:{}, c:{}, a:{}, f2:{} p:{}'.format(
favRel, ratRel, isNew, isComplete, authorRel, fandomRel, plain))
pfilter = ' '.join(plain).lower()
nfics: List[Fic] = []
completelyRefilter = (force or (self.filter[-1] == ' '
or self.filter[-1] == ':'))
# TODO FIXME bleh
userFics = {uf.ficId: uf for uf in UserFic.select({'userId': 1})}
for fic in (self.fics if completelyRefilter else self.list):
if fic.id not in userFics:
userFics[fic.id] = UserFic.default((1, fic.id))
userFic = userFics[fic.id]
if favRel is not None or ratRel is not None or isNew:
if favRel is not None:
if not self.fcmp(favRel[0], userFic.isFavorite, favRel[1]):
continue
if ratRel is not None:
if not self.fcmp(ratRel[0], userFic.rating or -1,
ratRel[1]):
continue
if isNew is not None:
if userFic.lastChapterViewed != 0:
continue
if descRel is not None:
if not self.fcmp(descRel[0], fic.description or '',
descRel[1]):
continue
if titleRel is not None:
if not self.fcmp(titleRel[0], fic.title or '', titleRel[1]):
continue
if isComplete is not None and fic.ficStatus != FicStatus.complete:
continue
if authorRel is not None:
if not self.fcmp(authorRel[0], fic.getAuthorName(),
authorRel[1]):
continue
if fandomRel is not None:
ficFandoms = [fandom.name for fandom in fic.fandoms()]
matchesFandom = False
for fandom in ficFandoms:
if self.fcmp(fandomRel[0], fandom, fandomRel[1]):
matchesFandom = True
break
if not matchesFandom:
continue
ftext = (
f'{fic.localId} {fic.title} {fic.getAuthorName()} {fic.id}'.
lower())
if util.subsequenceMatch(ftext, pfilter):
nfics += [fic]
self.list = nfics
