def test_basic_write_and_read(self):
(sentinel, schema_root) = self.__create_tree()
out = StringIO.StringIO()
writer = XMLTreeWriter(out)
writer.write(sentinel)
out_str = out.getvalue()
in_ = StringIO.StringIO(out_str)
reader = XMLTreeReader(in_)
new_sentinel = StandardNode()
reader.read(new_sentinel)
self.assert_(subtrees_equal(schema_root, new_sentinel.get_child("root")))
out2 = StringIO.StringIO()
writer2 = XMLTreeWriter(out2)
writer2.write(new_sentinel)
out2_str = out2.getvalue()
self.assertEqual(out_str, out2_str)
def test_basic_write_and_read(self):
(sentinel, schema_root) = self.__create_tree()
out = StringIO.StringIO()
writer = XMLTreeWriter(out)
writer.write(sentinel)
out_str = out.getvalue()
in_ = StringIO.StringIO(out_str)
reader = XMLTreeReader(in_)
new_sentinel = StandardNode()
reader.read(new_sentinel)
self.assert_(
subtrees_equal(schema_root, new_sentinel.get_child("root")))
out2 = StringIO.StringIO()
writer2 = XMLTreeWriter(out2)
writer2.write(new_sentinel)
out2_str = out2.getvalue()
self.assertEqual(out_str, out2_str)
def run(self):
args = self.__parse()
threads_no = args.threads
logging_level = self.__get_logging_level(args)
log_file_path = args.log_file
schedule = self.__get_schedule(args.daily_schedule)
navigators = self.__navigators_creator.create(args, threads_no)
sentinel = StandardNode()
prog = MultithreadedCrawler(navigators, sentinel, schedule,
log_file_path, args.state_file, self.__save_period, logging_level)
print "Starting activity with {} threads, "\
"activity daily schedule: {}".format(
threads_no, args.daily_schedule)
prog.run()
root = sentinel.get_child("root")
self.__navigators_creator.on_exit()
print "Done.\n"
print self.__get_tree_summary(root, args.state_file, log_file_path)
def run(self):
args = self.__parse()
threads_no = args.threads
logging_level = self.__get_logging_level(args)
log_file_path = args.log_file
schedule = self.__get_schedule(args.daily_schedule)
navigators = self.__navigators_creator.create(args, threads_no)
sentinel = StandardNode()
prog = MultithreadedCrawler(navigators, sentinel, schedule,
log_file_path, args.state_file,
self.__save_period, logging_level)
print "Starting activity with {} threads, "\
"activity daily schedule: {}".format(
threads_no, args.daily_schedule)
prog.run()
root = sentinel.get_child("root")
self.__navigators_creator.on_exit()
print "Done.\n"
print self.__get_tree_summary(root, args.state_file, log_file_path)
