def test_matching_initial_files_are_queued(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_two = ModelFile("File.Two", True)
file_two.remote_size = 200
file_three = ModelFile("File.Three", True)
file_three.remote_size = 300
file_four = ModelFile("File.Four", True)
file_four.remote_size = 400
file_five = ModelFile("File.Five", True)
file_five.remote_size = 500
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
calls = self.controller.queue_command.call_args_list
self.assertEqual(3, len(calls))
commands = [calls[i][0][0] for i in range(3)]
self.assertEqual(set([Controller.Command.Action.QUEUE] * 3),
{c.action
for c in commands})
self.assertEqual({"File.One", "File.Two", "File.Three"},
{c.filename
for c in commands})
def test_matching_queued_files_are_not_queued(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.state = ModelFile.State.QUEUED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_matching_new_files_are_extracted(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.state = ModelFile.State.DOWNLOADED
file_one.local_size = 100
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.state = ModelFile.State.DOWNLOADED
file_two.local_size = 200
file_two.is_extractable = True
file_three = ModelFile("File.Three", True)
file_three.state = ModelFile.State.DOWNLOADED
file_three.local_size = 300
file_three.is_extractable = True
self.model_listener.file_added(file_one)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.model_listener.file_added(file_two)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.Two", command.filename)
self.model_listener.file_added(file_three)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.Three", command.filename)
# All at once
self.model_listener.file_added(file_one)
self.model_listener.file_added(file_two)
self.model_listener.file_added(file_three)
auto_queue.process()
calls = self.controller.queue_command.call_args_list[-3:]
commands = [calls[i][0][0] for i in range(3)]
self.assertEqual(set([Controller.Command.Action.EXTRACT] * 3),
{c.action
for c in commands})
self.assertEqual({"File.One", "File.Two", "File.Three"},
{c.filename
for c in commands})
def test_partial_matches(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
self.model_listener.file_added(file_one)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
def test_matching_downloaded_files_are_not_queued(self):
# Disable auto-extract
self.context.config.autoqueue.auto_extract = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.state = ModelFile.State.DOWNLOADED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_non_extractable_files_are_not_extracted(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_two.is_extractable = False
self.initial_model = [file_one, file_two]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_all_files_are_extracted_when_patterns_only_disabled(self):
self.context.config.autoqueue.patterns_only = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_two.is_extractable = True
file_three = ModelFile("File.Three", True)
file_three.local_size = 300
file_three.state = ModelFile.State.DOWNLOADED
file_three.is_extractable = True
file_four = ModelFile("File.Four", True)
file_four.local_size = 400
file_four.state = ModelFile.State.DOWNLOADED
file_four.is_extractable = True
file_five = ModelFile("File.Five", True)
file_five.local_size = 500
file_five.state = ModelFile.State.DOWNLOADED
file_five.is_extractable = True
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
calls = self.controller.queue_command.call_args_list
self.assertEqual(5, len(calls))
commands = [calls[i][0][0] for i in range(5)]
self.assertEqual(set([Controller.Command.Action.EXTRACT] * 5),
{c.action
for c in commands})
self.assertEqual(
{"File.One", "File.Two", "File.Three", "File.Four", "File.Five"},
{c.filename
for c in commands})
def test_downloaded_file_with_changed_remote_size_is_queued(self):
# Disable auto-extract
self.context.config.autoqueue.auto_extract = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
file_one_updated = ModelFile("File.One", True)
file_one_updated.remote_size = 200
file_one_updated.local_size = 100
file_one_updated.state = ModelFile.State.DEFAULT
self.model_listener.file_updated(file_one, file_one_updated)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
def test_partial_file_is_auto_queued_after_remote_discovery(self):
# Test that a partial local file is auto-queued when discovered on remote some time later
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
# Local discovery
file_one = ModelFile("File.One", True)
file_one.local_size = 100
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
# Remote discovery
file_one_new = ModelFile("File.One", True)
file_one_new.local_size = 100
file_one_new.remote_size = 200
self.model_listener.file_updated(file_one, file_one_new)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
def test_downloaded_file_is_NOT_re_extracted_after_modified(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
# File is auto-extracted
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_one.is_extractable = True
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.EXTRACT, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
# File is modified
file_one_new = ModelFile("File.One", True)
file_one_new.local_size = 101
file_one_new.state = ModelFile.State.DOWNLOADED
file_one_new.is_extractable = True
self.model_listener.file_updated(file_one, file_one_new)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_adding_then_removing_pattern_doesnt_queue_existing_file(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_two = ModelFile("File.Two", True)
file_two.remote_size = 200
file_three = ModelFile("File.Three", True)
file_three.remote_size = 300
file_four = ModelFile("File.Four", True)
file_four.remote_size = 400
file_five = ModelFile("File.Five", True)
file_five.remote_size = 500
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.remove_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_no_files_are_extracted_when_auto_extract_disabled(self):
self.context.config.autoqueue.enabled = True
self.context.config.autoqueue.auto_extract = False
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
file_one = ModelFile("File.One", True)
file_one.local_size = 100
file_one.state = ModelFile.State.DOWNLOADED
file_two = ModelFile("File.Two", True)
file_two.local_size = 200
file_two.state = ModelFile.State.DOWNLOADED
file_three = ModelFile("File.Three", True)
file_three.local_size = 300
file_three.state = ModelFile.State.DOWNLOADED
file_four = ModelFile("File.Four", True)
file_four.local_size = 400
file_four.state = ModelFile.State.DOWNLOADED
file_five = ModelFile("File.Five", True)
file_five.local_size = 500
file_five.state = ModelFile.State.DOWNLOADED
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# First with patterns_only ON
self.context.config.autoqueue.patterns_only = True
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
# Second with patterns_only OFF
self.context.config.autoqueue.patterns_only = False
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_new_matching_pattern_doesnt_queue_local_file(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.local_size = 100
self.initial_model = [file_one]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_matching_downloading_files_are_not_queued(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_one.local_size = 0
file_one.state = ModelFile.State.DOWNLOADING
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_not_called()
file_one_new = ModelFile("File.One", True)
file_one_new.remote_size = 100
file_one_new.local_size = 50
file_one_new.state = ModelFile.State.DOWNLOADING
self.model_listener.file_updated(file_one, file_one_new)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def test_auto_queued_file_not_re_queued_after_stopping(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
self.model_listener.file_added(file_one)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
file_one_updated = ModelFile("File.One", True)
file_one_updated.remote_size = 100
file_one_updated.local_size = 50
self.model_listener.file_updated(file_one, file_one_updated)
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
def test_removed_pattern_doesnt_queue_new_file(self):
persist = AutoQueuePersist()
persist.add_pattern(AutoQueuePattern(pattern="One"))
persist.add_pattern(AutoQueuePattern(pattern="Two"))
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
self.model_listener.file_added(file_one)
auto_queue.process()
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
persist.remove_pattern(AutoQueuePattern(pattern="Two"))
file_two = ModelFile("File.Two", True)
file_two.remote_size = 100
self.model_listener.file_added(file_two)
auto_queue.process()
self.controller.queue_command.assert_not_called()
def run(self):
self.context.logger.info("Starting SeedSync")
self.context.logger.info("Platform: {}".format(platform.machine()))
# Create controller
controller = Controller(self.context, self.controller_persist)
# Create auto queue
auto_queue = AutoQueue(self.context, self.auto_queue_persist,
controller)
# Create web app
web_app_builder = WebAppBuilder(self.context, controller,
self.auto_queue_persist)
web_app = web_app_builder.build()
# Define child threads
controller_job = ControllerJob(
context=self.context.create_child_context(ControllerJob.__name__),
controller=controller,
auto_queue=auto_queue)
webapp_job = WebAppJob(context=self.context.create_child_context(
WebAppJob.__name__),
web_app=web_app)
do_start_controller = True
# Initial checks to see if we should bother starting the controller
if Seedsync._detect_incomplete_config(self.context.config):
if not self.context.args.exit:
do_start_controller = False
self.context.logger.error("Config is incomplete")
self.context.status.server.up = False
self.context.status.server.error_msg = Localization.Error.SETTINGS_INCOMPLETE
else:
raise AppError("Config is incomplete")
# Start child threads here
if do_start_controller:
controller_job.start()
webapp_job.start()
try:
prev_persist_timestamp = datetime.now()
# Thread loop
while True:
# Persist to file occasionally
now = datetime.now()
if (now - prev_persist_timestamp).total_seconds(
) > Constants.MIN_PERSIST_TO_FILE_INTERVAL_IN_SECS:
prev_persist_timestamp = now
self.persist()
# Propagate exceptions
webapp_job.propagate_exception()
# Catch controller exceptions and keep running, but notify the web server of the error
try:
controller_job.propagate_exception()
except AppError as exc:
if not self.context.args.exit:
self.context.status.server.up = False
self.context.status.server.error_msg = str(exc)
Seedsync.logger.exception("Caught exception")
else:
raise
# Check if a restart is requested
if web_app_builder.server_handler.is_restart_requested():
raise ServiceRestart()
# Nothing else to do
time.sleep(Constants.MAIN_THREAD_SLEEP_INTERVAL_IN_SECS)
except Exception:
self.context.logger.info("Exiting Seedsync")
# This sleep is important to allow the jobs to finish setup before we terminate them
# If we kill too early, the jobs may leave lingering threads around
# Note: There might be a better way to ensure that job setup has completed, but this
# will do for now
time.sleep(Constants.MAIN_THREAD_SLEEP_INTERVAL_IN_SECS)
# Join all the threads here
if do_start_controller:
controller_job.terminate()
webapp_job.terminate()
# Wait for the threads to close
if do_start_controller:
controller_job.join()
webapp_job.join()
# Last persist
self.persist()
# Raise any exceptions so they can be logged properly
# Note: ServiceRestart and ServiceExit will be caught and handled
# by outer code
raise
def test_new_matching_pattern_queues_existing_files(self):
persist = AutoQueuePersist()
file_one = ModelFile("File.One", True)
file_one.remote_size = 100
file_two = ModelFile("File.Two", True)
file_two.remote_size = 200
file_three = ModelFile("File.Three", True)
file_three.remote_size = 300
file_four = ModelFile("File.Four", True)
file_four.remote_size = 400
file_five = ModelFile("File.Five", True)
file_five.remote_size = 500
self.initial_model = [
file_one, file_two, file_three, file_four, file_five
]
# noinspection PyTypeChecker
auto_queue = AutoQueue(self.context, persist, self.controller)
auto_queue.process()
self.controller.queue_command.assert_not_called()
persist.add_pattern(AutoQueuePattern(pattern="File.One"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.One", command.filename)
self.controller.queue_command.reset_mock()
persist.add_pattern(AutoQueuePattern(pattern="File.Two"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.Two", command.filename)
self.controller.queue_command.reset_mock()
persist.add_pattern(AutoQueuePattern(pattern="File.Three"))
auto_queue.process()
self.controller.queue_command.assert_called_once_with(
unittest.mock.ANY)
command = self.controller.queue_command.call_args[0][0]
self.assertEqual(Controller.Command.Action.QUEUE, command.action)
self.assertEqual("File.Three", command.filename)
self.controller.queue_command.reset_mock()
auto_queue.process()
self.controller.queue_command.assert_not_called()
