def main(): disk.getFileLockOrDie("locks/backend.pid") options, args = parseArgs() if options.fix: urls = [url for url in storage.getUrls() if not storage.isDownloaded(url)] else: urls = tobe.getToBeDownloaded() logging.info("Downloading %s urls", len(urls)) for url in urls: content = _try_download(url) if content is not None: storage.storeContent(url, content) tobe.toAnalyse(url) tobe.nothingToBeDownloaded() logging.info("Downloaded %s urls", len(urls))
def main(): disk.getFileLockOrDie("locks/backend.pid") options, args = parseArgs() if options.fix: urls = [url for url in storage.getUrls() if storage.isDownloaded(url)] tobe.nothingToBeReported() if not options.clean: urls = _skipAnalysed(urls) else: urls = tobe.getToBeAnalysed() for url in urls: analysis = analyser.analyseUrl(url) storage.storeAnalysis(url, analysis) tobe.toReport(url) tobe.nothingToBeAnalysed() logging.info("Analysed %s urls", len(urls)) cleanStart = options.fix or options.clean updateReport(cleanStart) warmer.updateWarmPicture()