from lib.s3Storage import S3Storage
from lib.downloader import Downloader
from lib.formatter import Formatter
from lib.chambersApi import ChambersApi
logging.getLogger().setLevel(logging.INFO)
publications = ChambersApi.get_publications()
db = DB()
downloader = Downloader()
formatter = Formatter()
s3 = S3Storage("lcmillsconsulting.com", "guides")
try:
index_updated = True
for pub in publications:
if not db.has_publication(pub):
logging.info(
f"Found new publication: {pub['description']}, scraping...")
downloader.scrape(pub)
s3.upload_data(pub)
db.add_publication(pub)
logging.info("Scraped data")
if not db.is_publication_formatted(pub):
logging.info("Formatting html")
formatter.format_publication(pub)
s3.upload_report(pub)
db.set_publication_formatted(pub)
index_updated = True
logging.info("html Formatted")
