def crawl_set_time_main(key_word,start_time,end_time): thrads_list = [] while start_time+datetime.timedelta(days=30) < end_time: end_2 = start_time+datetime.timedelta(days=30) thrads_list.append(crawl_set_time_with_keyword(key_word,start_time,end_2,'crawl_settime_thread'+str(start_time)+" to "+str(end_2))) start_time = end_2 if start_time < end_time: thrads_list.append(crawl_set_time_with_keyword(key_word,start_time,end_time,'crawl_settime_thread'+str(start_time)+" to "+str(end_time))) return thrads_list
def crawl_set_time_main_many(key_word, start_time, end_time, how_many_days_one_thread, how_many_days_crawl_once): thrads_list = [] while start_time <= end_time: end_2 = start_time + datetime.timedelta(days=how_many_days_one_thread-1) thrads_list.append(crawl_set_time_with_keyword(key_word, start_time, end_2, how_many_days_crawl_once, 'crawl_settime_thread' + str(start_time) + " to " + str(end_2))) start_time = end_2+datetime.timedelta(days=1) return thrads_list
def crawl_set_time_main_many(key_word, start_time, end_time, how_many_days_one_thread, how_many_days_crawl_once): thrads_list = [] while start_time <= end_time: end_2 = start_time + datetime.timedelta(days=how_many_days_one_thread - 1) thrads_list.append( crawl_set_time_with_keyword( key_word, start_time, end_2, how_many_days_crawl_once, 'crawl_settime_thread' + str(start_time) + " to " + str(end_2))) start_time = end_2 + datetime.timedelta(days=1) return thrads_list