def crawl_set_time_main(key_word,start_time,end_time):
thrads_list = []
while start_time+datetime.timedelta(days=30) < end_time:
end_2 = start_time+datetime.timedelta(days=30)
thrads_list.append(crawl_set_time_with_keyword(key_word,start_time,end_2,'crawl_settime_thread'+str(start_time)+" to "+str(end_2)))
start_time = end_2
if start_time < end_time:
thrads_list.append(crawl_set_time_with_keyword(key_word,start_time,end_time,'crawl_settime_thread'+str(start_time)+" to "+str(end_time)))
return thrads_list
def crawl_set_time_main_many(key_word, start_time, end_time, how_many_days_one_thread, how_many_days_crawl_once):
thrads_list = []
while start_time <= end_time:
end_2 = start_time + datetime.timedelta(days=how_many_days_one_thread-1)
thrads_list.append(crawl_set_time_with_keyword(key_word, start_time, end_2, how_many_days_crawl_once, 'crawl_settime_thread' + str(start_time) + " to " + str(end_2)))
start_time = end_2+datetime.timedelta(days=1)
return thrads_list
def crawl_set_time_main_many(key_word, start_time, end_time,
how_many_days_one_thread,
how_many_days_crawl_once):
thrads_list = []
while start_time <= end_time:
end_2 = start_time + datetime.timedelta(days=how_many_days_one_thread -
1)
thrads_list.append(
crawl_set_time_with_keyword(
key_word, start_time, end_2, how_many_days_crawl_once,
'crawl_settime_thread' + str(start_time) + " to " +
str(end_2)))
start_time = end_2 + datetime.timedelta(days=1)
return thrads_list
