def main():
opts = parse_args()
log_to_stderr(verbose=opts.verbose, quiet=opts.quiet)
scraper_ids = opts.scraper_ids
if not scraper_ids and environ.get('MORPH_COMPANY_SCRAPERS'):
scraper_ids = environ['MORPH_COMPANY_SCRAPERS'].split(',')
skip_scraper_ids = DISABLED_SCRAPERS
if environ.get('MORPH_SKIP_COMPANY_SCRAPERS'):
skip_scraper_ids.update(
environ['MORPH_SKIP_COMPANY_SCRAPERS'].split(','))
use_decimal_type_in_sqlite()
run_scrapers(get_records_from_company_scraper,
scraper_ids=scraper_ids,
skip_scraper_ids=skip_scraper_ids,
default_freq=DEFAULT_SCRAPE_FREQ)
def main():
opts = parse_args()
level = logging.DEBUG if opts.verbose else logging.INFO
logging.basicConfig(format='%(name)s: %(message)s', level=level)
campaigns = opts.campaigns
if not campaigns and environ.get('MORPH_CAMPAIGNS'):
campaigns = environ['MORPH_CAMPAIGNS'].split(',')
skip_campaigns = set(DISABLED_CAMPAIGNS)
if environ.get('MORPH_SKIP_CAMPAIGNS'):
skip_campaigns.update(environ['MORPH_SKIP_CAMPAIGNS'].split(','))
use_decimal_type_in_sqlite()
run_scrapers(get_records_from_campaign_scraper,
scraper_ids=campaigns,
skip_scraper_ids=skip_campaigns,
default_freq=DEFAULT_SCRAPE_FREQ,
scraper_to_freq=CAMPAIGN_TO_SCRAPE_FREQ,
scraper_to_last_changed=CAMPAIGN_CHANGED_SINCE)
