def createFromZList(self, fic: Fic, ts: int, data: str) -> Fic: fic.url = self.constructUrl(fic.localId, 1) fic = self.parseZListInfoInto(fic, ts, data) fic.upsert() return Fic.lookup((fic.id, ))
def buildUrl(self, chapter: 'FicChapter') -> str: # TODO: do we need these 2 lines or will they always be done by however # FicChapter is created? if chapter.fic is None: chapter.fic = Fic.lookup((chapter.ficId, )) return self.constructUrl( chapter.fic.localId, chapter.chapterId, chapter.fic.title )
def buildUrl(self, chapter: 'FicChapter') -> str: # TODO: do we need these 2 lines or will they always be done by however # FicChapter is created? if chapter.fic is None: chapter.fic = Fic.lookup((chapter.ficId, )) if chapter.localChapterId is None: raise Exception('chapter missing localChapterId? FIXME') return self.constructUrl(chapter.fic.localId, int(chapter.localChapterId))
def getCurrentInfo(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId) url = self.tocUrl data = scrape.scrape(url) edumpContent('<!-- {} -->\n{}'.format(url, data['raw']), 'wavesarisen_ec') fic = self.parseInfoInto(fic, data['raw']) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId) data = scrape.softScrape(fic.url) if data is None: raise Exception('unable to scrape? FIXME') fic = self.parseInfoInto(fic, data) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId) # scrape fresh info data = scrape.scrape(fic.url) edumpContent(data['raw'], 'sugarquill') fic = self.parseInfoInto(fic, data['raw']) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: # TODO: should we try to get the actual url here, including the url safe # version of the title before the lid? Needs done elsewhere in this # adapter as well fic.url = self.baseUrl + 'threads/' + str(fic.localId) # scrape fresh info data = self.scrapeLike(fic.url) fic = self.parseInfoInto(fic, data) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId) # scrape fresh info data = scrape.scrape(fic.url) time.sleep(self.baseDelay) edumpContent(data['raw'], 'hpffa') fic = self.parseInfoInto(fic, data['raw']) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId, 1) # scrape fresh info data = scrape.scrape(fic.url) fic = self.parseInfoInto(fic, data['raw']) fic.insert() chapter = fic.chapter(1) chapter.setHtml(data['raw']) chapter.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.baseUrl + str(fic.localId) # scrape fresh info url = fic.url.split('?')[0] + '?view_adult=true' data = scrape.scrape(url) edumpContent(data['raw'], 'ao3') fic = self.parseInfoInto(fic, data['raw']) fic.upsert() chapter = fic.chapter(1) chapter.setHtml(data['raw']) chapter.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId, 1) # scrape fresh info data = scrape.softScrape(fic.url) if data is None: raise Exception('unable to scrape? FIXME') fic = self.parseInfoInto(fic, data) fic.upsert() chapter = fic.chapter(1) chapter.setHtml(data) chapter.localChapterId = str(1) chapter.url = self.constructUrl(fic.localId, 1) chapter.upsert() return Fic.lookup((fic.id, ))
def getCurrentInfo(self, fic: Fic) -> Fic: # grab the content from disk info = self.getArchiveStoryInfo(int(fic.localId)) spath = '{}/archive/{}/{}/summary.html.gz'.format( self.archivePath, info[1], info[2]) data = self.slurp(spath) fic = self.parseInfoInto(fic, data) fic.upsert() chapterCount = fic.chapterCount or 1 dCount = int(math.floor(math.log(chapterCount, 10) + 1)) localChapterIdMap = self.getChapterIds(int(fic.localId)) for cid in range(1, chapterCount + 1): pcid = str(cid).zfill(dCount) fpath = '{}/archive/{}/{}/chapters/chapter_{}.html.gz'.format( self.archivePath, info[1], info[2], pcid) data = self.slurp(fpath) chapter = fic.chapter(cid) chapter.localChapterId = localChapterIdMap[cid] chapter.setHtml(data) chapter.upsert() return Fic.lookup((fic.id, ))
def buildUrl(self, chapter: 'FicChapter') -> str: if chapter.fic is None: chapter.fic = Fic.lookup((chapter.ficId, )) return self.constructUrl(chapter.fic.localId, chapter.chapterId)