def test_shutdown(self):
clusters = [
MesosClusterRepository.get_cluster(address)
for address in ['a', 'b', 'c']
]
assert_equal(self.cluster_cls.call_count, 3)
MesosClusterRepository.shutdown()
for cluster in clusters:
assert_equal(cluster.stop.call_count, 1)
def test_shutdown(self):
clusters = [
MesosClusterRepository.get_cluster(address)
for address in ['a', 'b', 'c']
]
assert_equal(self.cluster_cls.call_count, 3)
MesosClusterRepository.shutdown()
for cluster in clusters:
assert_equal(cluster.stop.call_count, 1)
def restore_state(self, action_runner):
"""Use the state manager to retrieve to persisted state and apply it
to the configured Jobs.
"""
log.info('restoring')
states = self.state_watcher.restore(self.jobs.get_names())
MesosClusterRepository.restore_state(states.get('mesos_state', {}))
self.jobs.restore_state(states.get('job_state', {}), action_runner)
self.state_watcher.save_metadata()
def restore_state(self, action_runner):
"""Use the state manager to retrieve to persisted state and apply it
to the configured Jobs.
"""
log.info('restoring')
states = self.state_watcher.restore(self.jobs.get_names())
MesosClusterRepository.restore_state(states.get('mesos_state', {}))
self.jobs.restore_state(states.get('job_state', {}), action_runner)
self.state_watcher.save_metadata()
def _handle_shutdown(self, sig_num, stack_frame):
log.info(f"Shutdown requested via {str(sig_num)}")
reactor.callLater(0, reactor.stop)
waited = 0
while reactor.running:
if waited > 5:
log.error("timed out waiting for reactor shutdown")
break
time.sleep(0.1)
waited += 0.1
if self.mcp:
self.mcp.shutdown()
MesosClusterRepository.shutdown()
raise SystemExit(f"Terminating on signal {str(sig_num)}")
def _handle_shutdown(self, sig_num, stack_frame):
log.info("Shutdown requested via %s" % sig_num)
reactor.callLater(0, reactor.stop)
waited = 0
while reactor.running:
if waited > 5:
log.error("timed out waiting for reactor shutdown")
break
time.sleep(0.1)
waited += 0.1
if self.mcp:
self.mcp.shutdown()
MesosClusterRepository.shutdown()
self.context.terminate(sig_num, stack_frame)
def _handle_shutdown(self, sig_num, stack_frame):
log.info(f"Shutdown requested via {str(sig_num)}")
reactor.callLater(0, reactor.stop)
waited = 0
while reactor.running:
if waited > 5:
log.error("timed out waiting for reactor shutdown")
break
time.sleep(0.1)
waited += 0.1
if self.mcp:
self.mcp.shutdown()
MesosClusterRepository.shutdown()
raise SystemExit(f"Terminating on signal {str(sig_num)}")
def submit_command(self):
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = mesos_cluster.create_task(
action_run_id=self.id,
command=self.command,
cpus=self.cpus,
mem=self.mem,
constraints=self.constraints,
docker_image=self.docker_image,
docker_parameters=self.docker_parameters,
env=self.env,
extra_volumes=self.extra_volumes,
serializer=serializer,
)
if not task: # Mesos is disabled
self.fail(None)
return
self.mesos_task_id = task.get_mesos_id()
# Watch before submitting, in case submit causes a transition
self.watch(task)
mesos_cluster.submit(task)
return task
def recover(self):
if not self.machine.check('running'):
log.error(f'{self} unable to transition from {self.machine.state}'
'to running for recovery')
return
if self.mesos_task_id is None:
log.error(f'{self} no task ID, cannot recover')
self.fail_unknown()
return
log.info(f'{self} recovering Mesos run')
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = self._create_mesos_task(
mesos_cluster,
serializer,
self.mesos_task_id,
)
if not task:
log.warning(f'{self} cannot recover, Mesos is disabled or '
f'invalid task ID {self.mesos_task_id!r}')
self.fail_unknown()
return
self.watch(task)
mesos_cluster.recover(task)
# Reset status
self.exit_status = None
self.end_time = None
self.transition_and_notify('running')
return task
def test_configure(self):
clusters = [
MesosClusterRepository.get_cluster(address)
for address in ['d', 'e']
]
mock_volume = mock.Mock()
options = mock.Mock(
master_port=5000,
secret='/dev/null',
principal="fake-principal",
role='tron',
enabled=False,
default_volumes=[mock_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
with mock.patch(
'tron.mesos.get_secret_from_file',
autospec=True,
return_value='test-secret'
):
MesosClusterRepository.configure(options)
expected_volume = mock_volume._asdict.return_value
for cluster in clusters:
cluster.set_enabled.assert_called_once_with(False)
cluster.configure_tasks.assert_called_once_with(
default_volumes=[expected_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
# Next cluster we get should be initialized with the same settings
MesosClusterRepository.get_cluster('f')
self.cluster_cls.assert_called_with(
mesos_address='f',
mesos_master_port=5000,
secret='test-secret',
principal="fake-principal",
mesos_role='tron',
framework_id=None,
enabled=False,
default_volumes=[expected_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
def test_configure(self):
clusters = [
MesosClusterRepository.get_cluster(address)
for address in ['d', 'e']
]
mock_volume = mock.Mock()
options = mock.Mock(
master_port=5000,
secret='/dev/null',
principal="fake-principal",
role='tron',
enabled=False,
default_volumes=[mock_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
with mock.patch(
'tron.mesos.get_secret_from_file',
autospec=True,
return_value='test-secret'
):
MesosClusterRepository.configure(options)
expected_volume = mock_volume._asdict.return_value
for cluster in clusters:
cluster.set_enabled.assert_called_once_with(False)
cluster.configure_tasks.assert_called_once_with(
default_volumes=[expected_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
# Next cluster we get should be initialized with the same settings
MesosClusterRepository.get_cluster('f')
self.cluster_cls.assert_called_with(
mesos_address='f',
mesos_master_port=5000,
secret='test-secret',
principal="fake-principal",
mesos_role='tron',
framework_id=None,
enabled=False,
default_volumes=[expected_volume],
dockercfg_location='auth',
offer_timeout=1000,
)
def submit_command(self):
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = self._create_mesos_task(mesos_cluster, serializer)
if not task: # Mesos is disabled
self.fail(self.EXIT_MESOS_DISABLED)
return
self.mesos_task_id = task.get_mesos_id()
# Watch before submitting, in case submit causes a transition
self.watch(task)
mesos_cluster.submit(task)
return task
def submit_command(self):
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = self._create_mesos_task(mesos_cluster, serializer)
if not task: # Mesos is disabled
self.fail(self.EXIT_MESOS_DISABLED)
return
self.mesos_task_id = task.get_mesos_id()
# Watch before submitting, in case submit causes a transition
self.watch(task)
mesos_cluster.submit(task)
return task
def recover(self):
if self.mesos_task_id is None:
log.error(f'{self} no task ID, cannot recover')
return
if not self.machine.check('running'):
log.error(
f'{self} unable to transition from {self.machine.state}'
'to running for recovery'
)
return
log.info(f'{self} recovering Mesos run')
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = mesos_cluster.create_task(
action_run_id=self.id,
command=self.command,
cpus=self.cpus,
mem=self.mem,
constraints=self.constraints,
docker_image=self.docker_image,
docker_parameters=self.docker_parameters,
env=self.env,
extra_volumes=self.extra_volumes,
serializer=serializer,
task_id=self.mesos_task_id,
)
if not task:
log.warning(
f'{self} cannot recover, Mesos is disabled or '
f'invalid task ID {self.mesos_task_id!r}'
)
self.fail_unknown()
return
self.watch(task)
mesos_cluster.recover(task)
# Reset status
self.exit_status = None
self.end_time = None
self.transition_and_notify('running')
return task
def _kill_mesos_task(self):
msgs = []
if not self.is_active:
msgs.append(f'Action is {self.state}, not running. Continuing anyway.')
mesos_cluster = MesosClusterRepository.get_cluster()
if self.mesos_task_id is None:
msgs.append("Error: Can't find task id for the action.")
else:
msgs.append(f"Sending kill for {self.mesos_task_id}...")
succeeded = mesos_cluster.kill(self.mesos_task_id)
if succeeded:
msgs.append("Sent! It can take up to docker_stop_timeout (current setting is 2 mins) to stop.")
else:
msgs.append("Error while sending kill request. Please try again.")
return '\n'.join(msgs)
def recover(self):
if not self.machine.check('running'):
log.error(
f'{self} unable to transition from {self.machine.state}'
'to running for recovery'
)
return
if self.mesos_task_id is None:
log.error(f'{self} no task ID, cannot recover')
self.fail_unknown()
return
log.info(f'{self} recovering Mesos run')
serializer = filehandler.OutputStreamSerializer(self.output_path)
mesos_cluster = MesosClusterRepository.get_cluster()
task = self._create_mesos_task(
mesos_cluster,
serializer,
self.mesos_task_id,
)
if not task:
log.warning(
f'{self} cannot recover, Mesos is disabled or '
f'invalid task ID {self.mesos_task_id!r}'
)
self.fail_unknown()
return
self.watch(task)
mesos_cluster.recover(task)
# Reset status
self.exit_status = None
self.end_time = None
self.transition_and_notify('running')
return task
def _kill_mesos_task(self):
msgs = []
if not self.is_active:
msgs.append(
f'Action is {self.state}, not running. Continuing anyway.'
)
mesos_cluster = MesosClusterRepository.get_cluster()
if self.mesos_task_id is None:
msgs.append("Error: Can't find task id for the action.")
else:
msgs.append(f"Sending kill for {self.mesos_task_id}...")
succeeded = mesos_cluster.kill(self.mesos_task_id)
if succeeded:
msgs.append(
"Sent! It can take up to docker_stop_timeout (current setting is 2 mins) to stop."
)
else:
msgs.append(
"Error while sending kill request. Please try again."
)
return '\n'.join(msgs)
def test_get_cluster_repeated_mesos_address(self):
first = MesosClusterRepository.get_cluster('master-a.com')
second = MesosClusterRepository.get_cluster('master-a.com')
assert_equal(first, second)
assert_equal(self.cluster_cls.call_count, 1)
def test_get_cluster_repeated_mesos_address(self):
first = MesosClusterRepository.get_cluster('master-a.com')
second = MesosClusterRepository.get_cluster('master-a.com')
assert_equal(first, second)
assert_equal(self.cluster_cls.call_count, 1)