class StartCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, 'stopped',
'started')
copytool = models.ForeignKey(Copytool)
stateful_object = 'copytool'
state_verb = "Start"
display_group = Job.JOB_GROUPS.COMMON
display_order = 10
class Meta:
app_label = 'chroma_core'
ordering = ['id']
@classmethod
def get_args(cls, copytool):
return {'copytool_id': copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text['start_copytool']
def description(self):
return "Start copytool %s on worker %s" % (self.copytool,
self.copytool.host)
def get_steps(self):
return [(StartCopytoolStep, {
'host': self.copytool.host,
'copytool': self.copytool
})]
class ConfigureCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, 'unconfigured',
'stopped')
copytool = models.ForeignKey(Copytool)
stateful_object = 'copytool'
state_verb = "Configure"
display_group = Job.JOB_GROUPS.INFREQUENT
display_order = 10
class Meta:
app_label = 'chroma_core'
ordering = ['id']
@classmethod
def get_args(cls, copytool):
return {'copytool_id': copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text['configure_copytool']
def description(self):
return "Configure copytool %s on worker %s" % (self.copytool,
self.copytool.host)
def get_steps(self):
return [(ConfigureCopytoolStep, {
'host': self.copytool.host,
'filesystem': self.copytool.filesystem,
'client_mount': self.copytool.client_mount,
'copytool': self.copytool
})]
class RemoveStratagemJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(StratagemConfiguration,
"unconfigured",
"removed")
stateful_object = "stratagem_configuration"
stratagem_configuration = models.ForeignKey(StratagemConfiguration,
on_delete=CASCADE)
display_group = Job.JOB_GROUPS.COMMON
display_order = 10
requires_confirmation = False
state_verb = "Remove"
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
return "Remove Stratagem for the given filesystem"
def description(self):
return "Remove Stratagem for the given filesystem"
def get_steps(self):
return [(DeleteStratagemStep, {
"stratagem_configuration": self.stratagem_configuration
})]
def get_deps(self):
return DependOn(self.stratagem_configuration, "unconfigured")
class ForgetStratagemConfigurationJob(StateChangeJob):
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, self):
return "Forget the stratagem configuration for filesystem"
def description(self):
return self.long_description(self)
def get_requires_confirmation(self):
return True
def on_success(self):
self.stratagem_configuration.mark_deleted()
job_log.debug("forgetting stratagem configuration")
super(ForgetStratagemConfigurationJob, self).on_success()
state_transition = StateChangeJob.StateTransition(StratagemConfiguration,
"unconfigured",
"removed")
stateful_object = "stratagem_configuration"
stratagem_configuration = models.ForeignKey(StratagemConfiguration,
on_delete=CASCADE)
state_verb = "Forget"
class UnloadLNetJob(LNetStateChangeJob):
state_transition = StateChangeJob.StateTransition(LNetConfiguration, "lnet_down", "lnet_unloaded")
stateful_object = "lnet_configuration"
lnet_configuration = models.ForeignKey(LNetConfiguration, on_delete=CASCADE)
state_verb = "Unload LNet"
display_group = Job.JOB_GROUPS.RARE
display_order = 110
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
if stateful_object.is_managed:
return help_text["unload_lnet"]
else:
return help_text["Stop monitoring lnet on %s"] % stateful_object.host
def description(self):
return self.long_description(self.lnet_configuration)
def get_steps(self):
host_id = self.lnet_configuration.host.id
fqdn = self.lnet_configuration.host.fqdn
return self.lnet_configuration.filter_steps(
[
(UnloadLNetStep, {"fqdn": fqdn}),
(GetLNetStateStep, {"host_id": host_id, "fqdn": fqdn}),
]
)
class StartCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, "stopped",
"started")
copytool = models.ForeignKey(Copytool, on_delete=CASCADE)
stateful_object = "copytool"
state_verb = "Start"
display_group = Job.JOB_GROUPS.COMMON
display_order = 10
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def get_args(cls, copytool):
return {"copytool_id": copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text["start_copytool"]
def description(self):
return "Start copytool %s on worker %s" % (self.copytool,
self.copytool.host)
def get_steps(self):
return [(StartCopytoolStep, {
"host": self.copytool.host,
"copytool": self.copytool
})]
class EnableLNetJob(NullStateChangeJob):
target_object = models.ForeignKey(LNetConfiguration, on_delete=CASCADE)
state_transition = StateChangeJob.StateTransition(LNetConfiguration, "unconfigured", "lnet_unloaded")
class Meta:
app_label = "chroma_core"
ordering = ["id"]
def description(self):
return self.long_description(self.target_object)
@classmethod
def long_description(cls, stateful_object):
if stateful_object.is_managed:
return help_text["Enable LNet on %s"] % stateful_object.host
else:
return help_text["Start monitoring LNet on %s"] % stateful_object.host
def get_deps(self):
"""
Before LNet operations are possible some dependencies are need, basically the host must have had its packages installed.
Maybe we need a packages object, but this routine at least keeps the detail in one place.
Or maybe we need an unacceptable_states lists.
:return:
"""
if self.target_object.host.state in ["unconfigured", "undeployed"]:
return DependOn(self.target_object.host, "packages_installed")
else:
return DependAll()
class StopCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, "started", "stopped")
copytool = models.ForeignKey(Copytool)
stateful_object = "copytool"
state_verb = "Stop"
display_group = Job.JOB_GROUPS.COMMON
display_order = 10
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def get_args(cls, copytool):
return {"copytool_id": copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text["stop_copytool"]
def get_confirmation_string(self):
return StopCopytoolJob.long_description(None)
def get_requires_confirmation(self):
return True
def description(self):
return "Stop copytool %s on worker %s" % (self.copytool, self.copytool.host)
def get_steps(self):
return [
(CancelActiveOperationsStep, {"copytool": self.copytool}),
(StopCopytoolStep, {"host": self.copytool.host, "copytool": self.copytool}),
]
class RemoveLustreClientJob(StateChangeJob):
"""
Enables the client mount to be transitioned from unmounted -> removed
as part of a dependency resolution phase.
"""
state_transition = StateChangeJob.StateTransition(LustreClientMount,
"unmounted", "removed")
stateful_object = "lustre_client_mount"
lustre_client_mount = models.ForeignKey(LustreClientMount,
on_delete=CASCADE)
state_verb = None
@classmethod
def long_description(cls, stateful_object):
return help_text["remove_lustre_client_mount"]
def get_requires_confirmation(self):
return True
def get_confirmation_string(self):
return RemoveLustreClientJob.long_description(None)
def description(self):
return "Remove %s" % self.lustre_client_mount
def get_steps(self):
return [(DeleteLustreClientMountStep, {
"client_mount": self.lustre_client_mount
})]
class Meta:
app_label = "chroma_core"
ordering = ["id"]
class UnmountLustreClientMountJob(StateChangeJob):
"""
Enables the client mount to be transitioned from mounted -> unmounted
as part of a dependency resolution phase.
"""
state_transition = StateChangeJob.StateTransition(LustreClientMount, 'mounted', 'unmounted')
stateful_object = 'lustre_client_mount'
lustre_client_mount = models.ForeignKey(LustreClientMount)
state_verb = None
@classmethod
def long_description(cls, stateful_object):
return help_text['unmount_lustre_filesystem']
def get_requires_confirmation(self):
return True
def get_confirmation_string(self):
return UnmountLustreClientMountJob.long_description(None)
def description(self):
return "Unmount %s" % self.lustre_client_mount
def get_steps(self):
host = ObjectCache.get_one(ManagedHost, lambda mh: mh.id == self.lustre_client_mount.host_id)
from chroma_core.models.filesystem import ManagedFilesystem
filesystem = ObjectCache.get_one(ManagedFilesystem, lambda mf: mf.id == self.lustre_client_mount.filesystem_id)
args = dict(host = host,
filesystems = [(filesystem.mount_path(),
self.lustre_client_mount.mountpoint)])
return [(UnmountLustreFilesystemsStep, args)]
class Meta:
app_label = 'chroma_core'
ordering = ['id']
class ForgetLustreClientJob(StateChangeJob):
class Meta:
app_label = "chroma_core"
ordering = ["id"]
state_transition = StateChangeJob.StateTransition(LustreClientMount,
["unmounted", "mounted"],
"forgotten")
stateful_object = "lustre_client_mount"
state_verb = "Forget"
lustre_client_mount = models.ForeignKey(LustreClientMount,
on_delete=CASCADE)
requires_confirmation = True
@classmethod
def long_description(cls, stateful_object):
return "Forget this client mount on the manager. The actual client mount will not be altered."
def description(self):
return "Forget client mount {}".format(self.lustre_client_mount)
def on_success(self):
super(ForgetLustreClientJob, self).on_success()
self.lustre_client_mount.mark_deleted()
class RemoveCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, 'stopped',
'removed')
copytool = models.ForeignKey(Copytool)
stateful_object = 'copytool'
state_verb = "Remove"
display_group = Job.JOB_GROUPS.RARE
display_order = 10
class Meta:
app_label = 'chroma_core'
ordering = ['id']
@classmethod
def get_args(cls, copytool):
return {'copytool_id': copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text['remove_copytool']
def get_confirmation_string(self):
return RemoveCopytoolJob.long_description(None)
def get_requires_confirmation(self):
return True
def description(self):
return "Remove copytool %s on worker %s" % (self.copytool,
self.copytool.host)
def get_steps(self):
return [(CancelActiveOperationsStep, {
'copytool': self.copytool
}),
(StopCopytoolStep, {
'host': self.copytool.host,
'copytool': self.copytool
}),
(UnconfigureCopytoolStep, {
'host': self.copytool.host,
'copytool': self.copytool
}), (DeleteCopytoolStep, {
'copytool': self.copytool
})]
def get_deps(self):
search = lambda ct: ct.host == self.copytool.host
copytools = ObjectCache.get(Copytool, search)
# Only force an unmount if this is the only copytool associated
# with the host.
if len(copytools) == 1:
search = lambda cm: cm.id == self.copytool.client_mount_id
client_mount = ObjectCache.get_one(LustreClientMount, search)
return DependOn(client_mount, 'unmounted')
else:
return DependAll()
class MountLustreClientJob(StateChangeJob):
"""
Enables the client mount to be transitioned from unmounted -> mounted
as part of a dependency resolution phase.
"""
state_transition = StateChangeJob.StateTransition(LustreClientMount,
"unmounted", "mounted")
stateful_object = "lustre_client_mount"
lustre_client_mount = models.ForeignKey(LustreClientMount,
on_delete=CASCADE)
state_verb = None
@classmethod
def long_description(cls, stateful_object):
return help_text["mount_lustre_filesystem"]
def get_confirmation_string(self):
return MountLustreClientJob.long_description(None)
def description(self):
return "Mount %s" % self.lustre_client_mount
def get_steps(self):
from chroma_core.lib.graphql import get_client_mount_source
host = ManagedHost.objects.filter(
id=self.lustre_client_mount.host_id).values("fqdn").first()
mountspec = get_client_mount_source(
fs_name=self.lustre_client_mount.filesystem)
mountpoint = (self.lustre_client_mount.mountpoints[0]
if self.lustre_client_mount.mountpoints else
"/mnt/{}".format(self.lustre_client_mount.filesystem))
args = {
"host":
host.get("fqdn"),
"filesystems": [{
"mountspec": mountspec,
"mountpoint": mountpoint,
"persist": False
}],
}
return [(MountLustreFilesystemsStep, args)]
def get_deps(self):
return DependOn(
LNetConfiguration.objects.get(
host_id=self.lustre_client_mount.host_id), "lnet_up")
class Meta:
app_label = "chroma_core"
ordering = ["id"]
class MountLustreClientJob(StateChangeJob):
"""
Enables the client mount to be transitioned from unmounted -> mounted
as part of a dependency resolution phase.
"""
state_transition = StateChangeJob.StateTransition(LustreClientMount,
"unmounted", "mounted")
stateful_object = "lustre_client_mount"
lustre_client_mount = models.ForeignKey(LustreClientMount,
on_delete=CASCADE)
state_verb = None
@classmethod
def long_description(cls, stateful_object):
return help_text["mount_lustre_filesystem"]
def get_confirmation_string(self):
return MountLustreClientJob.long_description(None)
def description(self):
return "Mount %s" % self.lustre_client_mount
def get_steps(self):
host = ObjectCache.get_one(
ManagedHost, lambda mh: mh.id == self.lustre_client_mount.host_id)
mountpoint = (self.lustre_client_mount.mountpoints[0]
if self.lustre_client_mount.mountpoints else
"/mnt/{}".format(self.lustre_client_mount.filesystem))
filesystem = ObjectCache.get_one(
ManagedFilesystem,
lambda mf: mf.name == self.lustre_client_mount.filesystem)
args = {
"host": host,
"filesystems": [(filesystem.mount_path(), mountpoint)]
}
return [(MountLustreFilesystemsStep, args)]
def get_deps(self):
return DependOn(
ObjectCache.get_one(
ManagedHost, lambda mh: mh.id == self.lustre_client_mount.
host_id).lnet_configuration,
"lnet_up",
)
class Meta:
app_label = "chroma_core"
ordering = ["id"]
class UnmountLustreClientMountJob(StateChangeJob):
"""
Enables the client mount to be transitioned from mounted -> unmounted
as part of a dependency resolution phase.
"""
state_transition = StateChangeJob.StateTransition(LustreClientMount,
"mounted", "unmounted")
stateful_object = "lustre_client_mount"
lustre_client_mount = models.ForeignKey(LustreClientMount,
on_delete=CASCADE)
state_verb = None
@classmethod
def long_description(cls, stateful_object):
return help_text["unmount_lustre_filesystem"]
def get_requires_confirmation(self):
return True
def get_confirmation_string(self):
return UnmountLustreClientMountJob.long_description(None)
def description(self):
return "Unmount %s" % self.lustre_client_mount
def can_run(self):
client_mount = self.lustre_client_mount
return ManagedFilesystem.objects.filter(
name=client_mount.filesystem).exists()
def get_steps(self):
from chroma_core.lib.graphql import get_client_mount_source
client_mount = self.lustre_client_mount
host = ManagedHost.objects.filter(
id=client_mount.host_id).values("fqdn").first()
mount_spec = get_client_mount_source(fs_name=client_mount.filesystem)
filesystems = [{
"mountspec": mount_spec,
"mountpoint": x
} for x in client_mount.mountpoints]
args = {"host": host.get("fqdn"), "filesystems": filesystems}
return [(UnmountLustreFilesystemsStep, args)]
class Meta:
app_label = "chroma_core"
ordering = ["id"]
class RemoveUnconfiguredCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, "unconfigured",
"removed")
copytool = models.ForeignKey(Copytool, on_delete=CASCADE)
stateful_object = "copytool"
state_verb = "Remove"
display_group = Job.JOB_GROUPS.RARE
display_order = 10
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def get_args(cls, copytool):
return {"copytool_id": copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text["remove_copytool"]
def get_confirmation_string(self):
return RemoveUnconfiguredCopytoolJob.long_description(None)
def get_requires_confirmation(self):
return True
def description(self):
return "Remove copytool %s on worker %s" % (self.copytool,
self.copytool.host)
def get_steps(self):
return [(DeleteCopytoolStep, {"copytool": self.copytool})]
def get_deps(self):
search = lambda ct: ct.host == self.copytool.host
copytools = ObjectCache.get(Copytool, search)
# Only force an unmount if this is the only copytool associated
# with the host.
if len(copytools) == 1:
search = lambda cm: cm.id == self.copytool.client_mount_id
client_mount = ObjectCache.get_one(LustreClientMount, search)
return DependOn(client_mount, "unmounted")
else:
return DependAll()
class ForgetTargetJob(StateChangeJob):
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
return select_description(
stateful_object,
{
ManagedOst: help_text["remove_ost"],
ManagedMdt: help_text["remove_mdt"],
ManagedMgs: help_text["remove_mgt"],
},
)
def description(self):
modifier = "unmanaged" if self.target.immutable_state else "managed"
return "Forget %s target %s" % (modifier, self.target)
def get_requires_confirmation(self):
return True
def get_deps(self):
deps = []
if issubclass(self.target.downcast_class, ManagedMgs):
mgs = self.target.downcast()
ticket = mgs.get_ticket()
if ticket:
deps.append(
DependOn(ticket, "forgotten", fix_state=ticket.state))
return DependAll(deps)
def on_success(self):
_delete_target(self.target)
super(ForgetTargetJob, self).on_success()
state_transition = StateChangeJob.StateTransition(ManagedTarget,
["unmounted", "mounted"],
"forgotten")
stateful_object = "target"
state_verb = "Forget"
target = models.ForeignKey(ManagedTarget, on_delete=CASCADE)
class StopTargetJob(StateChangeJob):
stateful_object = "target"
state_transition = StateChangeJob.StateTransition(ManagedTarget, "mounted",
"unmounted")
state_verb = "Stop"
target = models.ForeignKey(ManagedTarget, on_delete=CASCADE)
def get_requires_confirmation(self):
return True
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
return select_description(
stateful_object,
{
ManagedOst: help_text["stop_ost"],
ManagedMgs: help_text["stop_mgt"],
ManagedMdt: help_text["stop_mdt"]
},
)
def description(self):
return "Stop target %s" % self.target
def get_deps(self):
if issubclass(self.target.downcast_class, ManagedMgs):
ticket = self.target.downcast().get_ticket()
if ticket:
return DependAll(
DependOn(ticket, "revoked", fix_state="mounted"))
return super(StopTargetJob, self).get_deps()
def get_steps(self):
return [
(UnmountStep, {
"fqdn": self.target.best_available_host().fqdn,
"ha_label": self.target.ha_label
}),
]
class UnconfigureLNetJob(NullStateChangeJob):
target_object = models.ForeignKey(LNetConfiguration, on_delete=CASCADE)
state_transition = StateChangeJob.StateTransition(LNetConfiguration, "lnet_unloaded", "unconfigured")
class Meta:
app_label = "chroma_core"
ordering = ["id"]
def description(self):
return self.long_description(self.target_object)
@classmethod
def long_description(cls, stateful_object):
if stateful_object.is_managed:
return help_text["Change lnet state of %s to unconfigured"] % stateful_object.host
else:
return help_text["Stop monitoring lnet on %s"] % stateful_object.host
def get_steps(self):
return self.target_object.filter_steps([(UnconfigureLNetStep, {"fqdn": self.target_object.host.fqdn})])
class ConfigureCopytoolJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(Copytool, "unconfigured", "stopped")
copytool = models.ForeignKey(Copytool)
stateful_object = "copytool"
state_verb = "Configure"
display_group = Job.JOB_GROUPS.INFREQUENT
display_order = 10
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def get_args(cls, copytool):
return {"copytool_id": copytool.pk}
@classmethod
def long_description(cls, stateful_object):
return help_text["configure_copytool"]
def description(self):
return "Configure copytool %s on worker %s" % (self.copytool, self.copytool.host)
def get_steps(self):
return [
(
ConfigureCopytoolStep,
{
"host": self.copytool.host,
"filesystem": self.copytool.filesystem,
"client_mount": self.copytool.client_mount,
"copytool": self.copytool,
},
)
]
class ConfigureStratagemJob(StateChangeJob):
state_transition = StateChangeJob.StateTransition(StratagemConfiguration,
"unconfigured",
"configured")
stateful_object = "stratagem_configuration"
stratagem_configuration = models.ForeignKey(StratagemConfiguration,
on_delete=CASCADE)
display_group = Job.JOB_GROUPS.COMMON
display_order = 10
requires_confirmation = False
state_verb = "Configure Stratagem"
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
return help_text["configure_stratagem_long"]
def description(self):
return help_text["configure_stratagem_long"]
def get_steps(self):
steps = []
if os.path.exists(timer_file(self.stratagem_configuration.id)):
steps.append((UnconfigureStratagemTimerStep, {
"config": self.stratagem_configuration
}))
steps.append((ConfigureStratagemTimerStep, {
"config": self.stratagem_configuration
}))
return steps
class StartTargetJob(StateChangeJob):
stateful_object = "target"
state_transition = StateChangeJob.StateTransition(ManagedTarget,
"unmounted", "mounted")
state_verb = "Start"
target = models.ForeignKey(ManagedTarget, on_delete=CASCADE)
class Meta:
app_label = "chroma_core"
ordering = ["id"]
@classmethod
def long_description(cls, stateful_object):
return select_description(
stateful_object,
{
ManagedOst: help_text["start_ost"],
ManagedMgs: help_text["start_mgt"],
ManagedMdt: help_text["start_mdt"],
},
)
def description(self):
return "Start target %s" % self.target
def get_deps(self):
if issubclass(self.target.downcast_class, ManagedMgs):
ticket = self.target.downcast().get_ticket()
if ticket:
return DependAll(
DependOn(ticket, "granted", fix_state="unmounted"))
if self.target.downcast_class in [ManagedMdt, ManagedOst]:
from chroma_core.models import FilesystemTicket
target = self.target.downcast()
ticket = FilesystemTicket.objects.filter(
filesystem=target.filesystem_id).first()
if ticket:
return DependAll(
DependOn(ticket.ticket, "granted", fix_state="unmounted"))
deps = []
# Depend on at least one targetmount having lnet up
for host in self.target.hosts:
from chroma_core.models import LNetConfiguration
lnet_configuration = ObjectCache.get_one(
LNetConfiguration, lambda l: l.host_id == host.id)
deps.append(
DependOn(lnet_configuration, "lnet_up", fix_state="unmounted"))
try:
pacemaker_configuration = ObjectCache.get_one(
PacemakerConfiguration, lambda pm: pm.host_id == host.id)
deps.append(
DependOn(pacemaker_configuration,
"started",
fix_state="unmounted"))
except PacemakerConfiguration.DoesNotExist:
pass
return DependAny(deps)
def get_steps(self):
return [(MountStep, {
"fqdn": self.target.best_available_host().fqdn,
"ha_label": self.target.ha_label
})]