formatter = Formatter()
s3 = S3Storage("lcmillsconsulting.com", "guides")
try:
index_updated = True
for pub in publications:
if not db.has_publication(pub):
logging.info(
f"Found new publication: {pub['description']}, scraping...")
downloader.scrape(pub)
s3.upload_data(pub)
db.add_publication(pub)
logging.info("Scraped data")
if not db.is_publication_formatted(pub):
logging.info("Formatting html")
formatter.format_publication(pub)
s3.upload_report(pub)
db.set_publication_formatted(pub)
index_updated = True
logging.info("html Formatted")
if index_updated:
formatter.create_index(db.get_all_publications())
s3.upload_index()
logging.info("Index updated")
logging.info("Done")
except Exception as e:
logging.error(e)
