#This little dance is because datetime returns
#the number days of the week as 0-6 and we want 1-7.
week_day_off_by_one = datetime.now().strftime('%w')
week_day_int = int(week_day_off_by_one)
week_day_int += 1
week_day = str(week_day_int)
#The lstrip is because we need to remove any leading 0's
month_day = datetime.now().strftime('%d').lstrip('0')
logging.info("Week day: " + week_day)
logging.info("Month day: " + month_day)
logging.info("Checking weekly batches.")
batches_weekly = ca.get_all_batches_to_crawl(week_day, 1)
if len(batches_weekly) == 0:
logging.info("No weekly batches scheduled to be crawled.")
else:
for batch_id in batches_weekly:
logging.info("Crawling batch " + batch_id)
os.system("python /var/www/tmeditor/python/recrawl_batch_generate_results.py " + batch_id + " \"" + secret_key + "\" \"" + access_id + "\" >> /tmp/crawl_generate_results.log 2>&1 &")
time.sleep(10)
logging.info("Checking daily batches.")
batches_daily = ca.get_all_batches_to_crawl(week_day, 3)
if len(batches_daily) == 0:
logging.info("No daily batches scheduled to be crawled.")
else:
for batch_id in batches_daily:
logging.info("Crawling batch " + batch_id)
