def test_spurious_item_events(self):
class StupidTask(SimpleTask):
def __init__(self):
SimpleTask.__init__(self, "StupidTask")
def process(self, item):
item.log_output('Failing the item.')
self.fail_item(item)
item.log_output('Completing the item.')
self.complete_item(item)
item.log_output('Failing the item.')
self.fail_item(item)
pipeline = Pipeline(StupidTask())
pipeline.fail_count_test = 0
def fail_callback(task, item):
pipeline.fail_count_test += 1
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertEqual(1, pipeline.fail_count_test)
self.assertIOLoopOK()
def test_spurious_item_events(self):
class StupidTask(SimpleTask):
def __init__(self):
SimpleTask.__init__(self, "StupidTask")
def process(self, item):
item.log_output('Failing the item.')
self.fail_item(item)
item.log_output('Completing the item.')
self.complete_item(item)
item.log_output('Failing the item.')
self.fail_item(item)
pipeline = Pipeline(StupidTask())
pipeline.fail_count_test = 0
def fail_callback(task, item):
pipeline.fail_count_test += 1
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertEqual(1, pipeline.fail_count_test)
self.assertIOLoopOK()
def test_runner_does_pipeline_cleanup_before_shutdown(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def cleanup():
self.cleanup_calls += 1
pipeline.on_cleanup += cleanup
runner.start()
self.assertEqual(1, self.cleanup_calls)
self.assertEqual(1, runner.item_count)
def test_runner_does_pipeline_cleanup_before_shutdown(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def cleanup():
self.cleanup_calls += 1
pipeline.on_cleanup += cleanup
runner.start()
self.assertEqual(1, self.cleanup_calls)
self.assertEqual(1, runner.item_count)
def test_runner_signals_pipeline_on_stop(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def stop_requested():
self.stop_requested_calls += 1
pipeline.on_stop_requested += stop_requested
runner.start()
runner.stop_gracefully()
self.assertEqual(1, self.stop_requested_calls)
def test_no_such_file(self):
external_process = ExternalProcessUser(
"Fake", ["kitteh and doge.avi.exe"])
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
def test_no_such_file(self):
external_process = ExternalProcessUser(
"Fake", ["kitteh and doge.avi.exe"])
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
def test_proc_utf8(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print(u'hello world!áßðfáßðf')"],
)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertIOLoopOK()
def test_proc(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print('hello world!')"], max_tries=4)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
output = external_process.output_buffer.getvalue()
self.assertFalse(pipeline.has_failed)
self.assertTrue('hello world!' in output)
self.assertIOLoopOK()
def test_proc_utf8(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print(u'hello world!áßðfáßðf')"],
)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertIOLoopOK()
def test_proc(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c", "print('hello world!')"], max_tries=4)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
output = external_process.output_buffer.getvalue()
self.assertFalse(pipeline.has_failed)
self.assertTrue('hello world!' in output)
self.assertIOLoopOK()
def test_runner_signals_pipeline_when_stop_canceled(self):
pipeline = Pipeline(PrintItem())
runner = SimpleRunner(pipeline, max_items=1)
def stop_canceled():
self.stop_canceled_calls += 1
pipeline.on_stop_canceled += stop_canceled
runner.start()
runner.stop_gracefully()
runner.keep_running()
self.assertEqual(1, self.stop_canceled_calls)
def test_proc_stdin_error(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c" "print('hello world!')"], max_tries=4)
external_process.stdin_data = lambda item: 123456
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
self.assertEqual(4, external_process.exit_count)
def test_proc_fail(self):
for max_tries in [1, 2, 20]:
external_process = ExternalProcessUser(
"Quitter", ["python", "-c", "import sys;sys.exit(33)"],
max_tries=max_tries)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertEqual(33, external_process.return_code)
self.assertEqual(max_tries, external_process.exit_count)
self.assertIOLoopOK()
def test_proc_stdin_error(self):
external_process = ExternalProcessUser(
"Echo", ["python", "-c" "print('hello world!')"], max_tries=4)
external_process.stdin_data = lambda item: 123456
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertIOLoopOK()
self.assertEqual(4, external_process.exit_count)
def test_proc_fail(self):
for max_tries in [1, 2, 20]:
external_process = ExternalProcessUser(
"Quitter", ["python", "-c", "import sys;sys.exit(33)"],
max_tries=max_tries)
pipeline = Pipeline(external_process)
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=1)
runner.start()
self.assertTrue(pipeline.has_failed)
self.assertEqual(33, external_process.return_code)
self.assertEqual(max_tries, external_process.exit_count)
self.assertIOLoopOK()
def test_max_items(self):
pipeline = Pipeline(PrintItem(), PrintItem())
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=3)
def finish_item_callback(runner, pipeline, item):
if runner.item_count > 10:
raise Exception('Too many items.')
runner.on_pipeline_finish_item += finish_item_callback
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertEqual(3, runner.item_count)
self.assertIOLoopOK()
def test_max_items(self):
pipeline = Pipeline(PrintItem(), PrintItem())
pipeline.has_failed = None
def fail_callback(task, item):
pipeline.has_failed = True
pipeline.on_fail_item += fail_callback
runner = SimpleRunner(pipeline, max_items=3)
def finish_item_callback(runner, pipeline, item):
if runner.item_count > 10:
raise Exception('Too many items.')
runner.on_pipeline_finish_item += finish_item_callback
runner.start()
self.assertFalse(pipeline.has_failed)
self.assertEqual(3, runner.item_count)
self.assertIOLoopOK()
def init_runner(args):
context = {"downloader": args.downloader}
for context_value in args.context_values:
name, text_value = context_value.split("=", 1)
if name not in context:
context[name] = text_value
else:
raise Exception("Context value name %s already defined." % name)
(project, pipeline) = load_pipeline(args.pipeline, context)
print("*" * 74)
print("*%-072s*" % " ")
print("*%-072s*" % (" ArchiveTeam Seesaw kit - %s" % seesaw.__version__))
print("*%-072s*" % " ")
print("*" * 74)
print()
print("Initializing pipeline for '%s'" % (project.title))
print()
print(pipeline)
print()
print("-" * 74)
print()
runner = SimpleRunner(pipeline,
stop_file=args.stop_file,
concurrent_items=args.concurrent_items,
max_items=args.max_items,
keep_data=args.keep_data)
if args.enable_web_server:
print("Starting the web interface on %s:%d..." %
(args.address, args.port_number))
print()
print("-" * 74)
print()
start_runner_server(project,
runner,
bind_address=args.address,
port_number=args.port_number,
http_username=args.http_username,
http_password=args.http_password)
print("Run 'touch %s' or interrupt (CTRL+C) to stop downloading." %
args.stop_file)
print()
attach_ctrl_c_handler(args.stop_file)
return runner
def main():
parser = ArgumentParser(description="Run the pipeline")
parser.add_argument("pipeline", metavar="PIPELINE", type=str,
help="the pipeline file")
parser.add_argument("downloader", metavar="DOWNLOADER", type=str,
help="your nickname")
parser.add_argument("--concurrent", dest="concurrent_items",
help="work on N items at a time (default: 1)",
metavar="N", type=int, default=1)
parser.add_argument("--max-items", dest="max_items",
help="stop after completing N items",
metavar="N", type=int, default=None)
parser.add_argument("--stop-file", dest="stop_file",
help="the STOP file to be monitored (default: STOP)",
metavar="FILE", type=str, default="STOP")
parser.add_argument("--disable-web-server", dest="enable_web_server",
help="disable the web interface",
action="store_false")
parser.add_argument("--keep-data", dest="keep_data",
help="do not remove data of finished items",
action="store_true")
parser.add_argument("--address", dest="address",
help="the IP address of the web interface "
"(default: 0.0.0.0)",
metavar="HOST", type=str, default="0.0.0.0")
parser.add_argument("--port", dest="port_number",
help="the port number for the web interface "
"(default: 8001)",
metavar="PORT", type=int, default=8001)
parser.add_argument("--http-username", dest="http_username",
help="username for the web interface (default: admin)",
metavar="USERNAME", type=str
) # default is set in start_runner_server
parser.add_argument("--http-password", dest="http_password",
help="password for the web interface",
metavar="PASSWORD", type=str)
parser.add_argument("--context-value", dest="context_values",
help="additional pipeline global variables "
"(name=text)",
metavar='VALUE_PAIR',
action='append', default=[], type=str)
parser.add_argument("--version", action="version",
version=seesaw.__version__)
args = parser.parse_args()
check_downloader_or_exit(args.downloader)
check_concurrency_or_exit(args.concurrent_items)
context = {"downloader": args.downloader}
for context_value in args.context_values:
name, text_value = context_value.split("=", 1)
if name not in context:
context[name] = text_value
else:
raise Exception("Context value name %s already defined." % name)
(project, pipeline) = load_pipeline(args.pipeline, context)
print("*" * 74)
print("*%-072s*" % " ")
print("*%-072s*" % (" ArchiveTeam Seesaw kit - %s" % seesaw.__version__))
print("*%-072s*" % " ")
print("*" * 74)
print()
print("Initializing pipeline for '%s'" % (project.title))
print()
print(pipeline)
print()
print("-" * 74)
print()
runner = SimpleRunner(
pipeline,
stop_file=args.stop_file,
concurrent_items=args.concurrent_items,
max_items=args.max_items,
keep_data=args.keep_data)
if args.enable_web_server:
print("Starting the web interface on %s:%d..." %
(args.address, args.port_number))
print()
print("-" * 74)
print()
start_runner_server(project, runner,
bind_address=args.address,
port_number=args.port_number,
http_username=args.http_username,
http_password=args.http_password)
print("Run 'touch %s' to stop downloading." % args.stop_file)
print()
runner.start()
