Esempio n. 1
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        parts = url.split('/')
        httpOrHttps = (parts[0] == 'https:' or parts[0] == 'http:')
        if len(parts) < 4:
            return None
        if (not parts[2].endswith(self.urlFragments[0])) or (not httpOrHttps):
            return None
        if not parts[3].startswith('story.php?'):
            return None

        leftover = parts[3].split('?')[-1]

        qs = urllib.parse.parse_qs(leftover)
        if 'no' not in qs or len(qs['no']) != 1:
            return None

        storyNumber = int(qs['no'][0])
        archive = parts[2].split('.')[0]
        lid = '{}/{}'.format(archive, storyNumber)

        ficId = FicId(self.ftype, lid)

        if 'chapter' in qs and len(qs['chapter']) == 1:
            ficId.chapterId = int(qs['chapter'][0])

        return ficId
Esempio n. 2
0
    def tryParseUrl(self, url: str) -> FicId:
        url = self.canonizeUrl(url)

        # if the url matches a chapter url, return it
        chapterUrls = self.getChapterUrls()
        if url in chapterUrls:
            return FicId(self.ftype, str(3), chapterUrls.index(url), False)

        # parahumans is id 3
        # TODO: change FicType.wavesarisen to wordpress?
        return FicId(self.ftype, str(3), ambiguous=False)
Esempio n. 3
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		# by default, we simply try to look up the url in existing chapters or fics
		chaps = FicChapter.select({'url': url})
		if len(chaps) == 1:
			fic = Fic.get((chaps[0].ficId, ))
			if fic is not None:
				return FicId(
					FicType(fic.sourceId), fic.localId, chaps[0].chapterId, False
				)

		fics = Fic.select({'url': url})
		if len(fics) == 1:
			return FicId(FicType(fics[0].sourceId), fics[0].localId)

		raise NotImplementedError()
Esempio n. 4
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		if not url.startswith(self.baseStoryUrl + '?'):
			return None
		qstring = url[len(self.baseStoryUrl + '?'):]

		qs = urllib.parse.parse_qs(qstring)
		if 'id' not in qs or len(qs['id']) != 1:
			return None

		lid = int(qs['id'][0])
		ficId = FicId(self.ftype, str(lid))

		if 'chapter' in qs and len(qs['chapter']) == 1:
			ficId.chapterId = int(qs['chapter'][0])

		return ficId
Esempio n. 5
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		if url.find('?') >= 0:
			url = url.split('?')[0]
		parts = url.split('/')
		httpOrHttps = (parts[0] == 'https:' or parts[0] == 'http:')
		if len(parts) < 5:
			return None
		if (not parts[2].endswith(self.urlFragments[0])) or (not httpOrHttps):
			return None
		if parts[3] != 's' and parts[3] != 'r':
			return None
		if (
			len(parts) < 5 or len(parts[4].strip()) < 1
			or not parts[4].strip().isnumeric()
		):
			return None

		storyId = int(parts[4])
		chapterId = None
		ambi = True
		if (
			len(parts) >= 6 and parts[3] == 's' and len(parts[5].strip()) > 0
			and parts[5].strip().isnumeric()
		):
			chapterId = int(parts[5].strip())
			ambi = False
		# upstream supports a chapter id after the story slug too, but it does not
		# normally generate such urls -- only use it as a fallback
		if (
			ambi and len(parts) >= 7 and parts[3] == 's' and len(parts[6].strip()) > 0
			and parts[6].strip().isnumeric()
		):
			chapterId = int(parts[6].strip())
			ambi = False
		return FicId(self.ftype, str(storyId), chapterId, ambi)
Esempio n. 6
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		if not url.startswith(self.baseStoryUrl):
			return None

		leftover = url[len(self.baseStoryUrl):]
		if not leftover.startswith('?'):
			return None
		leftover = leftover[1:]

		qs = urllib.parse.parse_qs(leftover)
		if 'storyid' not in qs or len(qs['storyid']) != 1:
			return None

		assert (qs['storyid'][0].isnumeric())
		ficId = FicId(self.ftype, qs['storyid'][0])

		if 'chapno' in qs and len(qs['chapno']) == 1:
			ficId.chapterId = int(qs['chapno'][0])

		return ficId
Esempio n. 7
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		parts = url.split('/')
		httpOrHttps = (parts[0] == 'https:' or parts[0] == 'http:')
		if len(parts) < 4:
			return None
		if (not parts[2].endswith(self.urlFragments[0])) or (not httpOrHttps):
			return None

		storyLid = parts[3]
		authorLid = parts[2].split('.')[0]
		lid = '{}/{}'.format(authorLid, storyLid)

		ficId = FicId(self.ftype, lid)

		if len(parts) > 4 and parts[4].startswith('Chapter_'):
			cid = int(parts[4][len('Chapter_'):])
			ficId.chapterId = cid
			ficId.ambiguous = False

		return ficId
Esempio n. 8
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		url = url.replace('royalroadl.com', 'royalroad.com')
		url = url.replace('https://royalroad.com', self.baseUrl)
		url = url.replace('http://', 'https://')
		if not url.startswith(self.baseStoryUrl):
			return None

		# TODO: is fiction a replacable category?
		# url like /fiction/{storyId}/{story name slug}
		# then optionally /chapter/{localChapterId}/{chapter title slug}
		rest = url[len(self.baseStoryUrl):]
		parts = rest.split('/')

		lid = int(parts[0])
		localChapterId = None

		if len(parts) > 2 and parts[1] == 'chapter':
			localChapterId = int(parts[2])
			return FicId(self.ftype, str(lid), localChapterId, False)

		return FicId(self.ftype, str(lid))
Esempio n. 9
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        stripSuffixes = ['&showRestricted']
        for suff in stripSuffixes:
            if url.lower().endswith(suff.lower()):
                url = url[:-len(suff)]
        mapPrefix = [
            ('http://', 'https://'),
            ('https://www.', 'https://'),
            ('https://fanfictionworld.net/hparchive', self.baseUrl),
            (self.baseUrl + '/viewstory2.php',
             self.baseUrl + '/viewstory.php'),
            (self.baseUrl + '/viewstory.php?sid=',
             self.baseUrl + '/viewstory.php?psid='),
            (self.baseUrl + '/reviews.php?storyid=',
             self.baseUrl + '/viewstory.php?psid='),
        ]
        for prefix in mapPrefix:
            if url.startswith(prefix[0]):
                url = prefix[1] + url[len(prefix[0]):]

        if not url.startswith(self.baseUrl):
            return None
        url = url[len(self.baseUrl):]

        if url.startswith(self.storyPrefix):
            rest = url[len(self.storyPrefix):]
            if rest.isnumeric():
                return FicId(self.ftype, rest)

        if not url.startswith(self.chapterPrefix):
            return None

        chapterId = url[len(self.chapterPrefix):]
        if not chapterId.isnumeric():
            return None

        info = self.getArchiveChapterInfo(int(chapterId))
        if info is None:
            return None
        return FicId(self.ftype, info[0], int(info[3]))
Esempio n. 10
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        if url.startswith("https://"):
            url = "http://" + url[len("https://"):]
        url = url.replace('http://hpfanficarchive.com',
                          'http://www.hpfanficarchive.com')
        if not url.startswith(self.baseStoryUrl):
            return None
        leftover = url[len(self.baseStoryUrl):]
        if not leftover.startswith('?'):
            return None
        leftover = leftover[1:]

        qs = urllib.parse.parse_qs(leftover)
        if 'sid' not in qs or len(qs['sid']) != 1:
            return None

        ficId = FicId(self.ftype, str(int(qs['sid'][0])))

        if 'chapter' in qs and len(qs['chapter']) == 1:
            ficId.chapterId = int(qs['chapter'][0])

        return ficId
Esempio n. 11
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		if not url.startswith(self.baseUrl):
			return None

		# by default, we simply try to look up the url in existing chapters or fics
		chaps = FicChapter.select({'url': url})
		if len(chaps) == 1:
			fic = Fic.get((chaps[0].ficId, ))
			if fic is not None:
				ftype = FicType(fic.sourceId)
				return FicId(ftype, fic.localId, chaps[0].chapterId, False)

		fics = Fic.select({'url': url})
		if len(fics) == 1:
			ftype = FicType(fics[0].sourceId)
			return FicId(ftype, fics[0].localId)

		leftover = url[len(self.baseUrl):]
		if not leftover.endswith('.html'):
			return None

		ps = leftover.split('/')
		if len(ps) != 3 or ps[0] != 'authors':
			return None

		author = ps[1]
		storyId = ps[2]
		suffixes = ['01a.html', '.html']
		for suffix in suffixes:
			if storyId.endswith(suffix):
				storyId = storyId[:-len(suffix)]

		# note: seems to be safe to lowercase these
		lid = (author + '/' + storyId).lower()
		#print(lid)
		# make lid author/story ?

		# TODO: we need some sort of local lid mapping...
		raise NotImplementedError()
Esempio n. 12
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        url = url.replace('&textsize=0', '')
        url = url.replace('http://', 'https://')
        url = url.replace('https://siye', 'https://www.siye')
        if url.startswith(self.alternateBaseUrl):
            url = self.baseUrl + url[len(self.alternateBaseUrl):]
        if not url.startswith(self.baseStoryUrl):
            return None

        leftover = url[len(self.baseStoryUrl):]
        if not leftover.startswith('?'):
            return None
        leftover = leftover[1:]

        qs = urllib.parse.parse_qs(leftover)
        if 'sid' not in qs or len(qs['sid']) != 1:
            return None

        ficId = FicId(self.ftype, str(int(qs['sid'][0])))

        if 'chapter' in qs and len(qs['chapter']) == 1:
            ficId.chapterId = int(qs['chapter'][0])

        return ficId
Esempio n. 13
0
def v0_lookup() -> Any:
    q = request.args.get('q', '').strip()
    if len(q.strip()) < 1:
        return Err.no_query.get({'arg': q})

    print(f'v0_lookup: query: {q}')
    ficId = FicId.tryParse(q)
    if ficId is None:
        return Err.bad_query.get({'arg': q})

    print(f'v0_lookup: ficId: {ficId.__dict__}')
    try:
        fic = Fic.load(ficId)
        return v0_fic(fic.urlId)
    except:
        print('v0_lookup: something went wrong in load:')
        traceback.print_exc()
        pass
    return Err.bad_ficId.get({'arg': ficId.__dict__})
Esempio n. 14
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        parts = url.split('/')
        httpOrHttps = (parts[0] == 'https:' or parts[0] == 'http:')
        if len(parts) < 4:
            return None
        if (not parts[2].endswith(self.urlFragments[0])) or (not httpOrHttps):
            return None
        if parts[3] != 'story':
            return None
        if (len(parts) < 5 or len(parts[4].strip()) < 1
                or not parts[4].strip().isnumeric()):
            return None

        storyId = int(parts[4])
        chapterId = None
        ambi = len(parts) < 6
        if ambi == False and len(parts[5].strip()) > 0:
            chapterId = int(parts[5])
        return FicId(self.ftype, str(storyId), chapterId, ambi)
Esempio n. 15
0
    def tryParseUrl(self, url: str) -> Optional[FicId]:
        if self.baseUrl.endswith('/'):
            url = url.replace(self.baseUrl + '/', self.baseUrl)
        for rw in self.rewrites:
            url = url.replace(rw[0], rw[1])
        parts = url.split('/')
        httpOrHttps = (parts[0] == 'https:' or parts[0] == 'http:')
        if len(parts) < 5:
            return None
        frag = parts[2]
        if self.urlFragments[0].find('/') >= 0:
            frag += '/' + parts[3]
        if (not frag.endswith(self.urlFragments[0])) or (not httpOrHttps):
            return None

        parts = url[len(self.baseUrl):].split('/')

        if parts[0] == 'posts':
            nurl = scrape.resolveRedirects(url)
            if nurl == url:
                raise Exception('unable to resolve redirects: {}'.format(url))
            return self.tryParseUrl(nurl)
        if not parts[0].endswith('threads'):
            return None
        if len(parts) < 2 or len(parts[1].strip()) < 1:
            return None

        storyId_s = parts[1]
        storyId_s = storyId_s.split('.')[-1]
        if not storyId_s.isnumeric():
            return None  # might not have a full id yet
        storyId = int(storyId_s)
        chapterId = None
        ambi = True  # TODO: ? len(parts) < 6
        if ambi == False and len(parts[2].strip()) > 0:
            chapterId = int(parts[2])
        return FicId(self.ftype, str(storyId), chapterId, ambi)
Esempio n. 16
0
    def extractSearchMetadata(
        self,
        html: str,
        metas: Dict[str, AdultFanfictionMeta] = {}
    ) -> Dict[str, AdultFanfictionMeta]:
        from bs4 import BeautifulSoup
        archiveFandomMap = {
            'naruto': 'Naruto',
            'hp': 'Harry Potter',
            'xmen': 'X-Men',
        }
        locatedFandomMap = [
            ('Mass Effect', 'Mass Effect'),
            ('Metroid', 'Metroid'),
            ('Pokemon', 'Pokemon'),
            ('Sonic', 'Sonic'),
            ('Witcher 3: Wild Hunt', 'Witcher'),
        ]
        chars = [
            'Harry', 'Hermione', 'Snape', 'Draco', 'Sirius', 'Remus', 'Lucius',
            'Ron', 'Voldemort', 'Ginny', 'Charlie', 'Lily', 'Scorpius',
            'James', 'George', 'Fred', 'Narcissa', 'Blaise', 'Bill', 'Luna',
            'Albus', 'Severus', 'Fenrir', 'Tonks', 'Rose', 'Neville', 'Cho',
            'Cedric', 'Tom', 'Seamus', 'Pansy', 'Bellatrix', 'Viktor', 'Percy',
            'Dudley', 'McGonagall', 'Lavendar', 'Dumbledore', 'Naruto',
            'Sasuke', 'Kakashi', 'Iruka', 'Sakura', 'Itachi', 'Gaara',
            'Shikamaru', 'Neji', 'Rock Lee', 'Hinata', 'Ino', 'Shino', 'Danzo'
        ]

        spaceSqeeezeRe = re.compile('\s+')

        searchSoup = BeautifulSoup(html, 'html5lib')
        resultTables = searchSoup.findAll('table', {'width': '90%'})
        for resultTable in resultTables:
            meta = AdultFanfictionMeta()

            links = resultTable.findAll('a')
            titleLink = links[0]
            meta.title = titleLink.getText()
            meta.url = titleLink.get('href')

            authorLink = links[1]
            meta.author = authorLink.getText().strip()
            meta.authorUrl = authorLink.get('href').strip()
            assert (meta.authorUrl is not None)
            meta.authorId = meta.authorUrl.split('=')[-1]

            trs = resultTable.findAll('tr')

            publishedText = trs[0].getText()
            RegexMatcher(publishedText, {
                'published': ('Published\s+:\s+(.+)', str),
            }).matchAll(meta)
            assert (meta.published is not None)
            meta.published = util.parseDateAsUnix(meta.published,
                                                  int(time.time()))

            extendedMetadata = trs[1].getText()
            util.logMessage(extendedMetadata, 'tmp_e_meta_aff.log')
            # TODO: dragon prints are actually views, not followCount/favoriteCount
            RegexMatcher(
                extendedMetadata, {
                    'chapterCount': ('Chapters\s*:\s*(\d+)', int),
                    'updated': ('Updated\s+:\s+(.+?)-:-', str),
                    'reviewCount?': ('Reviews\s+:\s+(\d+)', int),
                    'views?': ('Dragon prints\s+:\s+(\d+)', int),
                    'located?': ('Located\s*:\s*(.*)', str)
                }).matchAll(meta)
            assert (meta.updated is not None)
            meta.updated = util.parseDateAsUnix(meta.updated, int(time.time()))

            meta.description = str(trs[2])
            meta.description = util.filterUnicode(meta.description)
            meta.description = spaceSqeeezeRe.sub(' ', meta.description)

            meta.setTags(str(trs[3]))

            if 'COMPLETE' in meta.tags or 'Complete.' in meta.tags:
                meta.ficStatus = FicStatus.complete

            assert (meta.url is not None)
            ficId = FicId.tryParseUrl(meta.url)
            assert (ficId is not None)
            meta.localId = ficId.localId
            meta.archive = meta.localId.split('/')[0]
            meta.storyNo = meta.localId.split('/')[1]
            if meta.archive.lower() in archiveFandomMap:
                meta.fandoms += [archiveFandomMap[meta.archive.lower()]]

            meta.located = meta.located or ''
            loclow = meta.located.lower()

            for locFan in locatedFandomMap:
                if loclow.endswith(locFan[0].lower()):
                    meta.fandoms += [locFan[1]]

            for c1 in chars:
                for c2 in chars:
                    if loclow.endswith('{}/{}'.format(c1, c2).lower()):
                        meta.chars += [c1, c2]

            # TODO: try parse category, get chars
            #meta.info()

            if meta.url not in metas or meta.isNewerThan(metas[meta.url]):
                metas[meta.url] = meta

        return metas
Esempio n. 17
0
	def tryParseUrl(self, url: str) -> Optional[FicId]:
		mapPrefixes = ['http://www.', 'http://', 'https://www.']
		hasPrefix = True
		while hasPrefix:
			hasPrefix = False
			for pref in mapPrefixes:
				if url.startswith(pref):
					hasPrefix = True
					url = 'https://' + url[len(pref):]

		endsToStrip = [
			'#main',
			'#work_endnotes',
			'#bookmark-form',
			'?view_adult=true',
			'?view_full_work=true',
			'?viewfullwork=true',
			'?show_comments=true',
		]
		for send in endsToStrip:
			if url.endswith(send):
				url = url[:-len(send)]
		if url.find('#') >= 0:
			url = url[:url.find('#')]
		if url.find('?') >= 0:
			url = url[:url.find('?')]

		# TODO: this should probably return a FicId pointing to this chapter and
		# not just this fic in general...
		if url.find('/chapters/') >= 0 and url.find('/works/') < 0:
			meta = scrape.softScrapeWithMeta(url, delay=10)
			if meta is None or meta['raw'] is None or meta['status'] != 200:
				raise Exception('unable to lookup chapter: {}'.format(url))
			from bs4 import BeautifulSoup  # type: ignore
			soup = BeautifulSoup(meta['raw'], 'html5lib')
			for a in soup.find_all('a'):
				if a.get_text() == 'Entire Work':
					return self.tryParseUrl(self.baseUrl + a.get('href')[len('/works/'):])
			else:
				raise Exception('unable to lookup chapters entire work: {}'.format(url))

		if url.startswith(self.collectionUrl) and url.find('/works/') != -1:
			url = self.baseUrl + url[url.find('/works/') + len('/works/'):]
		if not url.startswith(self.baseUrl):
			return None

		pieces = url[len(self.baseUrl):].split('/')
		lid = pieces[0]
		if len(lid) < 1 or not lid.isnumeric():
			return None

		ficId = FicId(FicType.ao3, lid)
		fic = Fic.tryLoad(ficId)
		if fic is None:
			return ficId

		if len(pieces) >= 3 and pieces[1] == 'chapters' and pieces[2].isnumeric():
			localChapterId = pieces[2]
			mchaps = FicChapter.select(
				{
					'ficId': fic.id,
					'localChapterId': localChapterId
				}
			)
			if len(mchaps) == 1:
				ficId.chapterId = mchaps[0].chapterId
				ficId.ambiguous = False

		return ficId
Esempio n. 18
0
    def handleKey(self, key: int) -> bool:
        if key == 3:  # ctrl c
            if self.parent is not None:
                self.parent.quit()
            return True

        fic = None
        if len(self.list) > 0 and self.idx < len(self.list):
            fic = self.list[self.idx]

        if key == curses.KEY_DOWN:
            if self.idx < len(self.list) - 1:
                self.idx = (self.idx + 1) % len(self.list)
                return True
            return False
        if key == curses.KEY_UP:
            if self.idx == 0:
                return False
            self.idx = self.idx - 1
            return True
        if key == curses.KEY_HOME:
            self.idx = 0
            return True
        if key == curses.KEY_END:
            self.idx = len(self.list) - 1
            return True
        if key in {ord('\n'), ord('\r'), curses.KEY_ENTER, curses.KEY_RIGHT}:
            if len(self.list) == 0:
                ficId = FicId.parse(self.filter)
                fic = Fic.load(ficId)
                self.filter = ''
                self.fics = Fic.list()
                self.list = self.fics
                self.__refilter(fic)

                self.pushMessage('added fic "{}" ({})'.format(
                    fic.title, fic.localId))
            elif self.parent is not None:
                self.parent.selectFic(self.list[self.idx])
            return True

        if key == 4:  # ctrl d
            self.filter = ''
            self.list = self.fics
            self.__refilter(fic)
            return True

        # TODO: this is out of hand
        if ((key >= ord(' ') and key <= ord('~')) and
            ((chr(key).isalnum()) or ':/ .<>?&()=~'.find(chr(key)) != -1 or
             (len(self.filter) > 0 and key == ord('-')))):
            self.appendToFilter(chr(key).lower())
            return True
        if key in {curses.KEY_BACKSPACE, 127} and len(self.filter) > 0:
            self.backspace()
            return True
        if key == curses.KEY_PPAGE:
            self.idx = max(0, self.idx - int(self.height / 3))
            return True
        if key == curses.KEY_NPAGE:
            self.idx = min(len(self.list) - 1, self.idx + int(self.height / 3))
            return True

        if fic is None:
            return True
        userFic = self.getUserFic(fic)

        if key == 21:  # ctrl u
            userFic.lastChapterViewed = 0
            userFic.update()
            self.pushMessage('marked "{}" no last chapter'.format(fic.title))
            return True
        if key == 1 and fic.chapterCount is not None:  # ctrl a
            userFic.readStatus = FicStatus.complete
            userFic.updateLastViewed(fic.chapterCount)
            userFic.updateLastRead(fic.chapterCount)
            for cid in range(1, fic.chapterCount + 1):
                chap = fic.chapter(cid).getUserFicChapter()
                if chap.readStatus == FicStatus.complete:
                    continue
                chap.readStatus = FicStatus.complete
                chap.update()
            self.pushMessage('marked "{}" all read'.format(fic.title))
            return True
        if key == ord('+'):
            if userFic.rating is None or userFic.rating < 0: userFic.rating = 0
            if userFic.rating < 9:
                userFic.rating += 1
                userFic.update()
                self.pushMessage(
                    f'changed rating of "{fic.title}" => {userFic.rating}')
                return True
            return False
        if len(self.filter) == 0 and key == ord('-'):
            if userFic.rating is None: userFic.rating = 2
            if userFic.rating > 1:
                userFic.rating -= 1
                userFic.update()
                self.pushMessage(
                    f'changed rating of "{fic.title}" => {userFic.rating}')
                return True
            return False

        if key == 6:  # ctrl f
            userFic.isFavorite = not userFic.isFavorite
            userFic.update()
            self.pushMessage('changed favorite status of "{}"'.format(
                fic.title))
            return True
        if key == 9:  # ctrl i
            fic.checkForUpdates()
            self.pushMessage('checked "{}" for updates'.format(fic.title))
            return True
        if key == 23:  # ctrl w
            fic.ficStatus = {
                FicStatus.ongoing: FicStatus.abandoned,
                FicStatus.abandoned: FicStatus.complete,
                FicStatus.complete: FicStatus.ongoing,
            }[FicStatus(fic.ficStatus)]
            fic.upsert()
            return True
        return False
Esempio n. 19
0
 def fid(self) -> FicId:
     return FicId(FicType(self.sourceId), self.localId, ambiguous=False)
Esempio n. 20
0
    def __doRefilter(self, force: bool) -> None:
        if len(self.filter) < 1:
            return
        ficId = FicId.tryParse(self.filter)
        if ficId is not None:
            self.idx = 0
            self.list = []
            if ficId.ambiguous == False:
                fic = Fic.tryLoad(ficId)
                if fic is None:
                    fic = Fic.load(ficId)
                    self.fics = Fic.list()
                self.list = [fic]
            else:
                fic = Fic.tryLoad(ficId)
                if fic is not None:
                    self.list = [fic]
            return

        plain: List[str] = []
        tags: List[str] = []
        tagStarts = ['is:', 'i:', ':']
        for w in self.filter.split():
            isTag = False
            for tagStart in tagStarts:
                if w.startswith(tagStart):
                    tags += [w[len(tagStart):]]
                    isTag = True
                    break
            if not isTag:
                plain += [w]

        # TODO: simplify fcmp tags
        favRel = None
        ratRel = None
        isNew = None
        isComplete = None
        authorRel = None
        fandomRel = None
        descRel = None
        titleRel = None
        for tag in tags:
            if len(tag) == 0:
                continue
            arg: str = ''
            rel: Optional[str] = None
            for prel in ['=', '<', '>', '~', '.']:
                if tag.find(prel) != -1:
                    ps = tag.split(prel)
                    if len(ps) != 2:
                        continue
                    tag = ps[0]
                    rel = prel
                    arg = ps[1]
                    break

            if 'favorite'.startswith(tag):
                if rel is not None and arg.isnumeric():
                    favRel = (rel, int(arg))
                else:
                    favRel = ('>', 0)
            elif 'rated'.startswith(tag):
                if rel is not None and arg.isnumeric():
                    ratRel = (rel, int(arg))
                else:
                    ratRel = ('>', 0)
            elif 'new'.startswith(tag):
                isNew = ('=', 'new')
            elif 'author'.startswith(tag):
                if rel is not None:
                    authorRel = (rel, arg)
            elif 'fandom'.startswith(tag):
                if rel is not None:
                    fandomRel = (rel, arg)
            elif 'complete'.startswith(tag):
                isComplete = ('=', 'complete')
            elif 'description'.startswith(tag):
                if rel is not None:
                    descRel = (rel, arg)
            elif 'title'.startswith(tag):
                if rel is not None:
                    titleRel = (rel, arg)

        self.pushMessage('f:{}, r:{}, n:{}, c:{}, a:{}, f2:{} p:{}'.format(
            favRel, ratRel, isNew, isComplete, authorRel, fandomRel, plain))

        pfilter = ' '.join(plain).lower()

        nfics: List[Fic] = []
        completelyRefilter = (force or (self.filter[-1] == ' '
                                        or self.filter[-1] == ':'))

        # TODO FIXME bleh
        userFics = {uf.ficId: uf for uf in UserFic.select({'userId': 1})}

        for fic in (self.fics if completelyRefilter else self.list):
            if fic.id not in userFics:
                userFics[fic.id] = UserFic.default((1, fic.id))
            userFic = userFics[fic.id]
            if favRel is not None or ratRel is not None or isNew:
                if favRel is not None:
                    if not self.fcmp(favRel[0], userFic.isFavorite, favRel[1]):
                        continue
                if ratRel is not None:
                    if not self.fcmp(ratRel[0], userFic.rating or -1,
                                     ratRel[1]):
                        continue
                if isNew is not None:
                    if userFic.lastChapterViewed != 0:
                        continue
            if descRel is not None:
                if not self.fcmp(descRel[0], fic.description or '',
                                 descRel[1]):
                    continue
            if titleRel is not None:
                if not self.fcmp(titleRel[0], fic.title or '', titleRel[1]):
                    continue
            if isComplete is not None and fic.ficStatus != FicStatus.complete:
                continue
            if authorRel is not None:
                if not self.fcmp(authorRel[0], fic.getAuthorName(),
                                 authorRel[1]):
                    continue
            if fandomRel is not None:
                ficFandoms = [fandom.name for fandom in fic.fandoms()]
                matchesFandom = False
                for fandom in ficFandoms:
                    if self.fcmp(fandomRel[0], fandom, fandomRel[1]):
                        matchesFandom = True
                        break
                if not matchesFandom:
                    continue

            ftext = (
                f'{fic.localId} {fic.title} {fic.getAuthorName()} {fic.id}'.
                lower())
            if util.subsequenceMatch(ftext, pfilter):
                nfics += [fic]
        self.list = nfics