def createFromZList(self, fic: Fic, ts: int, data: str) -> Fic: fic.url = self.constructUrl(fic.localId, 1) fic = self.parseZListInfoInto(fic, ts, data) fic.upsert() return Fic.lookup((fic.id, ))
def buildUrl(self, chapter: 'FicChapter') -> str: # TODO: do we need these 2 lines or will they always be done by however # FicChapter is created? if chapter.fic is None: chapter.fic = Fic.lookup((chapter.ficId, )) return self.constructUrl( chapter.fic.localId, chapter.chapterId, chapter.fic.title )
def buildUrl(self, chapter: 'FicChapter') -> str:
# TODO: do we need these 2 lines or will they always be done by however
# FicChapter is created?
if chapter.fic is None:
chapter.fic = Fic.lookup((chapter.ficId, ))
if chapter.localChapterId is None:
raise Exception('chapter missing localChapterId? FIXME')
return self.constructUrl(chapter.fic.localId,
int(chapter.localChapterId))
def getCurrentInfo(self, fic: Fic) -> Fic:
fic.url = self.constructUrl(fic.localId)
url = self.tocUrl
data = scrape.scrape(url)
edumpContent('<!-- {} -->\n{}'.format(url, data['raw']),
'wavesarisen_ec')
fic = self.parseInfoInto(fic, data['raw'])
fic.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
fic.url = self.constructUrl(fic.localId)
data = scrape.softScrape(fic.url)
if data is None:
raise Exception('unable to scrape? FIXME')
fic = self.parseInfoInto(fic, data)
fic.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic: fic.url = self.constructUrl(fic.localId) # scrape fresh info data = scrape.scrape(fic.url) edumpContent(data['raw'], 'sugarquill') fic = self.parseInfoInto(fic, data['raw']) fic.upsert() return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
# TODO: should we try to get the actual url here, including the url safe
# version of the title before the lid? Needs done elsewhere in this
# adapter as well
fic.url = self.baseUrl + 'threads/' + str(fic.localId)
# scrape fresh info
data = self.scrapeLike(fic.url)
fic = self.parseInfoInto(fic, data)
fic.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
fic.url = self.constructUrl(fic.localId)
# scrape fresh info
data = scrape.scrape(fic.url)
time.sleep(self.baseDelay)
edumpContent(data['raw'], 'hpffa')
fic = self.parseInfoInto(fic, data['raw'])
fic.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
fic.url = self.constructUrl(fic.localId, 1)
# scrape fresh info
data = scrape.scrape(fic.url)
fic = self.parseInfoInto(fic, data['raw'])
fic.insert()
chapter = fic.chapter(1)
chapter.setHtml(data['raw'])
chapter.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
fic.url = self.baseUrl + str(fic.localId)
# scrape fresh info
url = fic.url.split('?')[0] + '?view_adult=true'
data = scrape.scrape(url)
edumpContent(data['raw'], 'ao3')
fic = self.parseInfoInto(fic, data['raw'])
fic.upsert()
chapter = fic.chapter(1)
chapter.setHtml(data['raw'])
chapter.upsert()
return Fic.lookup((fic.id, ))
def create(self, fic: Fic) -> Fic:
fic.url = self.constructUrl(fic.localId, 1)
# scrape fresh info
data = scrape.softScrape(fic.url)
if data is None:
raise Exception('unable to scrape? FIXME')
fic = self.parseInfoInto(fic, data)
fic.upsert()
chapter = fic.chapter(1)
chapter.setHtml(data)
chapter.localChapterId = str(1)
chapter.url = self.constructUrl(fic.localId, 1)
chapter.upsert()
return Fic.lookup((fic.id, ))
def getCurrentInfo(self, fic: Fic) -> Fic:
# grab the content from disk
info = self.getArchiveStoryInfo(int(fic.localId))
spath = '{}/archive/{}/{}/summary.html.gz'.format(
self.archivePath, info[1], info[2])
data = self.slurp(spath)
fic = self.parseInfoInto(fic, data)
fic.upsert()
chapterCount = fic.chapterCount or 1
dCount = int(math.floor(math.log(chapterCount, 10) + 1))
localChapterIdMap = self.getChapterIds(int(fic.localId))
for cid in range(1, chapterCount + 1):
pcid = str(cid).zfill(dCount)
fpath = '{}/archive/{}/{}/chapters/chapter_{}.html.gz'.format(
self.archivePath, info[1], info[2], pcid)
data = self.slurp(fpath)
chapter = fic.chapter(cid)
chapter.localChapterId = localChapterIdMap[cid]
chapter.setHtml(data)
chapter.upsert()
return Fic.lookup((fic.id, ))
def buildUrl(self, chapter: 'FicChapter') -> str: if chapter.fic is None: chapter.fic = Fic.lookup((chapter.ficId, )) return self.constructUrl(chapter.fic.localId, chapter.chapterId)
