def handle_execute(status):
"""
This function handles incoming responses for the method `execute`. The
corresponding `ProgramStatusModel` will be modified and the user will be
notified.
Parameters
----------
status: Status
The `Status` object that was send by the slave
"""
LOGGER.info("Handle program execute %s", dict(status))
try:
program_status = ProgramStatusModel.objects.get(
command_uuid=status.uuid)
program = program_status.program
except ProgramStatusModel.DoesNotExist:
LOGGER.warning(
"A program finished with id %s, but is not in the database.",
status.uuid,
)
return
LOGGER.info(
"Received answer on execute request of function %s from %s.",
program.name,
program.slave.name,
)
if status.is_ok():
LOGGER.info(
"Saved status of %s with code %s.",
program.name,
program_status.code,
)
else:
LOGGER.error(
'Exception in occurred client while executing %s: %s %s',
program.name,
os.linesep,
status.payload['result'],
)
# update status
program_status.code = status.payload['result']
program_status.running = False
program_status.save()
# tell webinterface that the program has ended
notify({
'program_status': 'finished',
'pid': str(program.id),
'code': status.payload['result']
})
def handle_filesystem_moved(status):
"""
This function handles incoming responses for the method `filesystem_move`.
The corresponding `FilesystemModel` will be modified and the user will be
notified.
Parameters
----------
status: Status
The `Status` object that was send by the slave
"""
LOGGER.info("Handle filesystem moved %s", dict(status))
try:
file_ = FilesystemModel.objects.get(command_uuid=status.uuid)
except FilesystemModel.DoesNotExist:
LOGGER.warning(
"A filesystem moved with id %s, but is not in the database.",
status.uuid,
)
return
if status.is_ok():
file_.hash_value = status.payload['result']
file_.error_code = ""
file_.save()
LOGGER.info(
"Saved filesystem %s with hash value %s.",
file_.name,
file_.hash_value,
)
notify({
'filesystem_status': 'moved',
'fid': str(file_.id),
})
else:
file_.error_code = status.payload['result']
file_.hash_value = ""
file_.save()
LOGGER.error(
"Error while moving filesystem: %s",
status.payload['result'],
)
notify({
'filesystem_status': 'error',
'error_code': status.payload['result'],
'fid': str(file_.id),
})
def ws_rpc_disconnect(message):
"""
Handles disconnects for websockets on `/commands`. The disconnect can only
be successful if the sender is a known client. If the sender is a known
client then the `SlaveModel` will be updated and the sender will be removed
from the group. The user will be notified if the disconnect was successful.
Parameters
----------
message: channels.message.Message
The last message which is send by the sender.
"""
try:
slave = SlaveModel.objects.get(
ip_address=message.channel_session['ip_address'])
Group('client_{}'.format(slave.id)).discard(message.reply_channel)
slave.online = False
slave.command_uuid = None
slave.save()
# if a slave disconnects all programs stop
for program in ProgramModel.objects.filter(slave=slave):
if ProgramStatusModel.objects.filter(program=program).exists():
ProgramStatusModel.objects.get(program=program).delete()
# tell the web interface that the client has disconnected
notify({'slave_status': 'disconnected', 'sid': str(slave.id)})
# notify the scheduler that status has change
FSIM_CURRENT_SCHEDULER.notify()
LOGGER.info(
"Client with ip %s disconnected from /commands!",
message.channel_session['ip_address'],
)
except SlaveModel.DoesNotExist:
LOGGER.error(
"Disconnected client is not in database. (with IP %s)",
message.channel_session['ip_address'],
)
def handle_get_log(status):
"""
This function handles incoming responses for the method `get_log`.
The corresponding `ProgramStatusModel` will be modified and the user will
be notified.
Parameters
----------
status: Status
The `Status` object that was send by the slave
"""
LOGGER.info("Handle log get %s", dict(status))
if status.is_ok():
try:
program_status = ProgramStatusModel.objects.get(
command_uuid=status.payload['result']['uuid'])
program = program_status.program
except ProgramStatusModel.DoesNotExist:
notify_err('Received log from unknown program!')
LOGGER.warning(
"A log from a programm with id %s has arrived, but is not in the database.",
status.uuid,
)
return
LOGGER.info(
"Received answer on get_log request of program %s on %s.",
program.name,
program.slave.name,
)
notify({
'log': status.payload['result']['log'],
'pid': str(program.id)
})
LOGGER.info("Send log of %s to the webinterface", program.name)
else:
notify_err('An error occured while reading a log file!')
LOGGER.error(
'Exception occurred (get_log-request): %s %s',
os.linesep,
status.payload['result'],
)
def slave_wake_on_lan(slave):
"""
This functions starts a `slave` by sending a magic (Wake-On-Lan
package) to the `slave`.
Parameters
----------
slave: SlaveModel
A valid `SlaveModel`.
Raises
------
TypeError:
If `slave` is not an `SlaveModel`
"""
ensure_type("slave", slave, SlaveModel)
send_magic_packet(slave.mac_address)
notify({"message": "Send start command to client `{}`".format(slave.name)})
def slave_shutdown(slave):
"""
This functions shutsdown a `slave` by a command to the slave.
Parameters
----------
slave: SlaveModel
A valid `SlaveModel`.
Raises
------
TypeError:
If `slave` is not an `SlaveModel`
"""
if slave.is_online:
notify_slave(Command(method="shutdown"), slave.id)
notify({"message": "Send shutdown Command to {}".format(slave.name)})
else:
raise SlaveOfflineError('', '', 'shutdown', slave.name)
def __state_error(self):
"""
This function handles the `ERROR` state, where the `Scheduler` finishes
with an error.
"""
from .models import Script
LOGGER.info("Scheduler is finished. (ERROR)")
Script.objects.filter(id=self.__script).update(
is_running=False,
error_code=self.__error_code,
)
notify({
'script_status': 'error',
'error_code': self.__error_code,
'script_id': self.__script,
})
def handle_online(status):
"""
This function handles incoming responses for the method `online`.
The corresponding `SlaveModel` will be modified and the user will be
notified.
Parameters
----------
status: Status
The `Status` object that was send by the slave
"""
LOGGER.info("Handle slave online %s", dict(status))
try:
slave = SlaveModel.objects.get(command_uuid=status.uuid)
except SlaveModel.DoesNotExist:
LOGGER.warning(
"Slaves online request with uuid %s, was not asked for it.",
status.uuid,
)
return
if status.is_ok():
slave.online = True
slave.save()
# tell webinterface that the client has been connected
notify({'slave_status': 'connected', 'sid': str(slave.id)})
LOGGER.info(
'Slave %s has connected to the master',
slave.name,
)
else:
# notify the webinterface
notify_err('An error occurred while connecting to client {}!'.format(
slave.name))
LOGGER.error(
'Exception occurred in client %s (online-request): %s %s',
slave.name,
os.linesep,
status.payload['result'],
)
def __state_success(self):
"""
This function handles the `SUCCESS` state, where the `Scheduler`
finishes without an error.
"""
from .models import Script
LOGGER.info("Scheduler is finished. (SUCCESS)")
notify({
'script_status': 'success',
'script_id': self.__script,
})
Script.objects.filter(id=self.__script).update(
is_running=False,
error_code='',
)
Script.set_last_started(self.__script)
def __state_init(self):
"""
This functions handles the `INIT` state. And sending every relevant
slave the Wake-On-Lan package.
"""
from .models import Script, Slave
from .controller import slave_wake_on_lan
for slave in Script.get_involved_slaves(self.__script):
LOGGER.debug("Send WOL to the slave `%s`.", slave.name)
slave_wake_on_lan(slave)
self.__state = SchedulerStatus.WAITING_FOR_SLAVES
self.__event.set()
self.loop.spawn(300, self.slave_timeout_callback)
notify({
'script_status': 'waiting_for_slaves',
'script_id': self.__script,
})
def prog_start(prog):
"""
This functions starts a `prog` by sending a command to the slave.
The program can only be started if the program is currently not running.
If the slave is offline an error will be returned.
Parameters
----------
prog: ProgramModel
A valid `ProgramModel`.
Raises
------
SlaveOfflineError
ProgramRunningError
TypeError:
If `prog` is not an `ProgramModel`
"""
ensure_type("prog", prog, ProgramModel)
if prog.slave.is_online:
if prog.is_running:
raise ProgramRunningError(str(prog.name), str(prog.slave.name))
uuid = uuid4().hex
cmd = Command(
uuid=uuid, # for the command
pid=prog.id,
own_uuid=uuid, # for the function that gets executed
method="execute",
path=prog.path,
arguments=[prog.arguments],
)
LOGGER.info(
"Starting program %s on slave %s",
prog.name,
prog.slave.name,
)
# send command to the client
notify_slave(cmd, prog.slave.id)
# tell webinterface that the program has started
notify({
'program_status': 'started',
'pid': prog.id,
})
# create status entry
ProgramStatusModel(program=prog,
command_uuid=cmd.uuid,
start_time=now()).save()
if prog.start_time > 0:
LOGGER.debug(
'started timeout on %s, for %d seconds',
prog.name,
prog.start_time,
)
LOGGER.debug(type(prog.start_time))
FSIM_CURRENT_SCHEDULER.spawn(
prog.start_time,
timer_timeout_program,
prog.id,
)
elif prog.start_time == 0:
timer_timeout_program(prog.id)
else:
raise SlaveOfflineError(
str(prog.name),
"program",
str(prog.slave.name),
"start",
)
def __state_next(self):
"""
This function handles the `NEXT_STEP` state, where all programs are
started and all filesystems are moved. If a program is started then it
will be not started again. If a filesystem is moved already then it
will be not moved again.
"""
from .controller import prog_start, fs_move
from .errors import (
FilesystemMovedError,
ProgramRunningError,
SlaveOfflineError,
)
from .models import (
Script,
ScriptGraphPrograms,
ScriptGraphFiles,
)
(last_index, all_done) = self.__get_next_stage()
max_start_time = 0
LOGGER.info(
"Starting programs and moving files for stage `%s`",
self.__index,
)
Script.objects.filter(id=self.__script).update(
current_index=self.__index)
if all_done:
LOGGER.info(
"Could not find another index after `%s` ... done.",
last_index,
)
self.__state = SchedulerStatus.SUCCESS
self.__event.set()
else:
notify_me = False
for sgp in ScriptGraphPrograms.objects.filter(
script=self.__script,
index=self.__index,
):
try:
if sgp.program.start_time > max_start_time:
max_start_time = sgp.program.start_time
prog_start(sgp.program)
LOGGER.info("Started program `%s`", sgp.program.name)
except ProgramRunningError as err:
LOGGER.info("Program `%s` is already started.", err.name)
notify_me = True
continue
except SlaveOfflineError as err:
LOGGER.error(
"A slave is gone offline while the execution.")
self.__state = SchedulerStatus.ERROR
self.__error_code = str(err)
self.__event.set()
return
for sgf in ScriptGraphFiles.objects.filter(
script=self.__script,
index=self.__index,
):
try:
fs_move(sgf.filesystem)
LOGGER.info("Moved filesystem `%s`", sgf.filesystem.name)
except FilesystemMovedError as err:
# if the filesystem is already moved go on.
LOGGER.info("Filesystem `%s` is already moved.", err.name)
notify_me = True
except SlaveOfflineError as err:
LOGGER.error(
"A slave is gone offline while the execution.")
self.__state = SchedulerStatus.ERROR
self.__error_code = str(err)
self.__event.set()
return
LOGGER.info(
"Started all programs for stage `%s`.",
self.__index,
)
self.__state = SchedulerStatus.WAITING_FOR_PROGRAMS_FILESYSTEMS
if notify_me:
LOGGER.info(
"Notify myself because some entries are already ready.")
self.__event.set()
notify({
'script_status': 'next_step',
'index': self.__index,
'last_index': last_index,
'start_time': max_start_time,
'script_id': self.__script,
})