def _go(self): fl = FeedLoader() fl.go() fl.closeDB() if not runStatus.run: return cl = ContentLoader() cl.go() cl.closeDB() if __name__ == "__main__": import utilities.testBase as tb with tb.testSetup(): # run = Runner() # run.go() runStatus.run = True cl = ContentLoader() cl.go() cl.closeDB() # cl = ContentLoader() # pg = 'http://www.twistedhelscans.com/read/trace_15/en/0/33/page/1' # pg = 'http://www.twistedhelscans.com/read/trace_15/en/0/32/' # cl.getImageUrls(pg)
itemList = self.getItemPages(url, title)
for item in itemList:
ret.append(item)
if not runStatus.run:
self.log.info("Breaking due to exit flag being set")
break
self.log.info("Found %s total items", len(ret))
return ret
def go(self):
self.resetStuckItems()
self.log.info("Getting feed items")
feedItems = self.getAllItems()
self.log.info("Processing feed Items")
self.processLinksIntoDB(feedItems)
self.log.info("Complete")
if __name__ == "__main__":
print('wat')
import utilities.testBase as tb
with tb.testSetup(startObservers=False):
cl = FeedLoader()
cl.go()
return ret def setup(self): # Muck about in the webget internal settings self.wg.errorOutCount = 4 self.wg.retryDelay = 5 def get_feed(self): treedata = self.wg.getJson(self.tree_api) assert 'contents' in treedata assert treedata['name'] == 'mango' assert treedata['type'] == 'directory' data_unfiltered = self.process_tree_elements(treedata['contents']) return data_unfiltered if __name__ == "__main__": import utilities.testBase as tb with tb.testSetup(load=False): run = FeedLoader() # run.go() run.do_fetch_feeds()
from ScrapePlugins.NHentaiLoader.DbLoader import DbLoader from ScrapePlugins.NHentaiLoader.ContentLoader import ContentLoader import runStatus import ScrapePlugins.RunBase class Runner(ScrapePlugins.RunBase.ScraperBase): loggerPath = "Main.Manga.NHentai.Run" pluginName = "NHentai" sourceName = "NHentai" feedLoader = DbLoader contentLoader = ContentLoader if __name__ == "__main__": import utilities.testBase as tb with tb.testSetup(startObservers=False): run = Runner() run.go()
fl.go()
fl.closeDB()
time.sleep(3)
#print "wat", cl
if not runStatus.run:
return
cl = ContentLoader()
if not runStatus.run:
return
todo = cl.retreiveTodoLinksFromDB()
if not runStatus.run:
return
cl.processTodoLinks(todo)
cl.closeDB()
if __name__ == '__main__':
import utilities.testBase as tb
with tb.testSetup():
fl = Runner()
fl.go()
@abc.abstractmethod
def loggerPath(self):
return None
def __init__(self):
self.db = db
self.log = logging.getLogger("Main.Trigger."+self.loggerPath)
self.log.info("Loading %s Runner", self.pluginName)
def _go(self):
self.log.info("Checking %s for updates", self.pluginName)
self.go()
self.log.info("Update check for %s finished.", self.pluginName)
if __name__ == "__main__":
import utilities.testBase as tb
with tb.testSetup(startObservers=True):
run = Runner()
run.go()
starturl=url,
netloc=urllib.parse.urlsplit(url).netloc,
priority=self.db.DB_HIGH_PRIORITY,
distance=0)
sess.add(new)
sess.commit()
break
except sqlalchemy.exc.InternalError:
self.log.info("Transaction error. Retrying.")
sess.rollback()
except sqlalchemy.exc.OperationalError:
self.log.info("Transaction error. Retrying.")
sess.rollback()
except sqlalchemy.exc.IntegrityError:
self.log.info("Transaction error. Retrying.")
sess.rollback()
except sqlalchemy.exc.InvalidRequestError:
self.log.info("Transaction error. Retrying.")
sess.rollback()
if __name__ == "__main__":
import utilities.testBase as tb
with tb.testSetup(startObservers=True):
run = Runner()
run.go()
def test():
import utilities.testBase as tb
with tb.testSetup(startObservers=False):
run = Runner()
run.go()
def test(): import utilities.testBase as tb with tb.testSetup(startObservers=False): run = Runner() run.go()
items = self.loadFeed(pageOverride)
ret = []
for feedEntry in items:
item = {}
try:
item["dlName"] = feedEntry["name"]
item["contentID"] = feedEntry["token"]
item["date"] = calendar.timegm(parser.parse( feedEntry["date"]).utctimetuple())
#self.log.info("date = ", feedEntry['published_parsed'])
item['retreivalTime'] = time.time()
ret.append(item)
except:
self.log.info("WAT?")
traceback.print_exc()
return ret
if __name__ == "__main__":
import utilities.testBase as tb
with tb.testSetup(load=False):
# getHistory()
run = DjMoeDbLoader()
# run.getFeed()
run.go()
def test(): import utilities.testBase as tb with tb.testSetup(): run = Runner() run.go()
def test(): import utilities.testBase as tb with tb.testSetup(): run = Runner() run.go()