def main(): opts = parse_args() log_to_stderr(verbose=opts.verbose, quiet=opts.quiet) scraper_ids = opts.scraper_ids if not scraper_ids and environ.get('MORPH_COMPANY_SCRAPERS'): scraper_ids = environ['MORPH_COMPANY_SCRAPERS'].split(',') skip_scraper_ids = DISABLED_SCRAPERS if environ.get('MORPH_SKIP_COMPANY_SCRAPERS'): skip_scraper_ids.update( environ['MORPH_SKIP_COMPANY_SCRAPERS'].split(',')) use_decimal_type_in_sqlite() run_scrapers(get_records_from_company_scraper, scraper_ids=scraper_ids, skip_scraper_ids=skip_scraper_ids, default_freq=DEFAULT_SCRAPE_FREQ)
def main(): opts = parse_args() level = logging.DEBUG if opts.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) campaigns = opts.campaigns if not campaigns and environ.get('MORPH_CAMPAIGNS'): campaigns = environ['MORPH_CAMPAIGNS'].split(',') skip_campaigns = set(DISABLED_CAMPAIGNS) if environ.get('MORPH_SKIP_CAMPAIGNS'): skip_campaigns.update(environ['MORPH_SKIP_CAMPAIGNS'].split(',')) use_decimal_type_in_sqlite() run_scrapers(get_records_from_campaign_scraper, scraper_ids=campaigns, skip_scraper_ids=skip_campaigns, default_freq=DEFAULT_SCRAPE_FREQ, scraper_to_freq=CAMPAIGN_TO_SCRAPE_FREQ, scraper_to_last_changed=CAMPAIGN_CHANGED_SINCE)