def test_spurious_item_events(self):
class StupidTask(SimpleTask):
def __init__(self):
SimpleTask.__init__(self, "StupidTask")
def process(self, item):
item.log_output('Failing the item.')
self.fail_item(item)
item.log_output('Completing the item.')
self.complete_item(item)
item.log_output('Failing the item.')
self.fail_item(item)
pipeline = Pipeline(StupidTask())
pipeline.fail_count_test = 0
def fail_callback(task, item):
pipeline.fail_count_test += 1
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertEqual(1, pipeline.fail_count_test)
self.assertIOLoopOK()
def test_runner_signals_pipeline_on_stop(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def stop_requested():
self.stop_requested_calls += 1
pipeline.on_stop_requested += stop_requested
runner.start()
runner.stop_gracefully()
self.assertEqual(1, self.stop_requested_calls)
def test_runner_does_pipeline_cleanup_before_shutdown(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def cleanup():
self.cleanup_calls += 1
pipeline.on_cleanup += cleanup
runner.start()
self.assertEqual(1, self.cleanup_calls)
self.assertEqual(1, runner.item_count)
def init_runner(args):
context = {"downloader": args.downloader}
for context_value in args.context_values:
name, text_value = context_value.split("=", 1)
if name not in context:
context[name] = text_value
else:
raise Exception("Context value name %s already defined." % name)
(project, pipeline) = load_pipeline(args.pipeline, context)
print("*" * 74)
print("*%-072s*" % " ")
print("*%-072s*" % (" ArchiveTeam Seesaw kit - %s" % seesaw.__version__))
print("*%-072s*" % " ")
print("*" * 74)
print()
print("Initializing pipeline for '%s'" % (project.title))
print()
print(pipeline)
print()
print("-" * 74)
print()
runner = SimpleRunner(pipeline,
stop_file=args.stop_file,
concurrent_items=args.concurrent_items,
max_items=args.max_items,
keep_data=args.keep_data)
if args.enable_web_server:
print("Starting the web interface on %s:%d..." %
(args.address, args.port_number))
print()
print("-" * 74)
print()
start_runner_server(project,
runner,
bind_address=args.address,
port_number=args.port_number,
http_username=args.http_username,
http_password=args.http_password)
print("Run 'touch %s' or interrupt (CTRL+C) to stop downloading." %
args.stop_file)
print()
attach_ctrl_c_handler(args.stop_file)
return runner
def test_no_such_file(self):
external_process = ExternalProcessUser(
"Fake", ["kitteh and doge.avi.exe"])
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
def test_proc(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print('hello world!')"], max_tries=4)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
output = external_process.output_buffer.getvalue()
self.assertFalse(pipeline.has_failed)
self.assertTrue('hello world!' in output)
self.assertIOLoopOK()
def test_proc_utf8(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print(u'hello world!áßðfáßðf')"],
)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertIOLoopOK()
def test_proc_stdin_error(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c" "print('hello world!')"], max_tries=4)
external_process.stdin_data = lambda item: 123456
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
self.assertEqual(4, external_process.exit_count)
def test_proc_fail(self):
for max_tries in [1, 2, 20]:
external_process = ExternalProcessUser(
"Quitter", ["python", "-c", "import sys;sys.exit(33)"],
max_tries=max_tries)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertEqual(33, external_process.return_code)
self.assertEqual(max_tries, external_process.exit_count)
self.assertIOLoopOK()
def test_max_items(self):
pipeline = Pipeline(PrintItem(), PrintItem())
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=3)
def finish_item_callback(runner, pipeline, item):
if runner.item_count > 10:
raise Exception('Too many items.')
runner.on_pipeline_finish_item += finish_item_callback
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertEqual(3, runner.item_count)
self.assertIOLoopOK()
