def __init__(self, navigators, sentinel, activity_schedule=None, log_file_path=None, state_file_path=None, save_period=None, logging_level=logging.ERROR): MultithreadedCrawler.__init__(self, navigators, sentinel, activity_schedule, log_file_path, state_file_path, save_period, logging_level)
def run(self): args = self.__parse() threads_no = args.threads logging_level = self.__get_logging_level(args) log_file_path = args.log_file schedule = self.__get_schedule(args.daily_schedule) navigators = self.__navigators_creator.create(args, threads_no) sentinel = StandardNode() prog = MultithreadedCrawler(navigators, sentinel, schedule, log_file_path, args.state_file, self.__save_period, logging_level) print "Starting activity with {} threads, "\ "activity daily schedule: {}".format( threads_no, args.daily_schedule) prog.run() root = sentinel.get_child("root") self.__navigators_creator.on_exit() print "Done.\n" print self.__get_tree_summary(root, args.state_file, log_file_path)