def __init__(self,
navigators,
sentinel,
activity_schedule=None,
log_file_path=None,
state_file_path=None,
save_period=None,
logging_level=logging.ERROR):
MultithreadedCrawler.__init__(self, navigators, sentinel,
activity_schedule, log_file_path,
state_file_path, save_period,
logging_level)
def run(self):
args = self.__parse()
threads_no = args.threads
logging_level = self.__get_logging_level(args)
log_file_path = args.log_file
schedule = self.__get_schedule(args.daily_schedule)
navigators = self.__navigators_creator.create(args, threads_no)
sentinel = StandardNode()
prog = MultithreadedCrawler(navigators, sentinel, schedule,
log_file_path, args.state_file, self.__save_period, logging_level)
print "Starting activity with {} threads, "\
"activity daily schedule: {}".format(
threads_no, args.daily_schedule)
prog.run()
root = sentinel.get_child("root")
self.__navigators_creator.on_exit()
print "Done.\n"
print self.__get_tree_summary(root, args.state_file, log_file_path)
def run(self):
args = self.__parse()
threads_no = args.threads
logging_level = self.__get_logging_level(args)
log_file_path = args.log_file
schedule = self.__get_schedule(args.daily_schedule)
navigators = self.__navigators_creator.create(args, threads_no)
sentinel = StandardNode()
prog = MultithreadedCrawler(navigators, sentinel, schedule,
log_file_path, args.state_file,
self.__save_period, logging_level)
print "Starting activity with {} threads, "\
"activity daily schedule: {}".format(
threads_no, args.daily_schedule)
prog.run()
root = sentinel.get_child("root")
self.__navigators_creator.on_exit()
print "Done.\n"
print self.__get_tree_summary(root, args.state_file, log_file_path)
def __init__(self, navigators, sentinel, activity_schedule=None, log_file_path=None, state_file_path=None, save_period=None, logging_level=logging.ERROR): MultithreadedCrawler.__init__(self, navigators, sentinel, activity_schedule, log_file_path, state_file_path, save_period, logging_level)
