def _GetGroupData(qcl, uuid): """Retrieves instances and nodes per node group. """ queries = [ (constants.QR_INSTANCE, [ "name", "status", "disks_active", "snodes", "pnode.group.uuid", "snodes.group.uuid" ], [qlang.OP_EQUAL, "pnode.group.uuid", uuid]), (constants.QR_NODE, ["name", "bootid", "offline"], [qlang.OP_EQUAL, "group.uuid", uuid]), ] results = [] for what, fields, qfilter in queries: results.append(qcl.Query(what, fields, qfilter)) results_data = map(operator.attrgetter("data"), results) # Ensure results are tuples with two values assert compat.all( map(ht.TListOf(ht.TListOf(ht.TIsLength(2))), results_data)) # Extract values ignoring result status (raw_instances, raw_nodes) = [[map(compat.snd, values) for values in res] for res in results_data] secondaries = {} instances = [] # Load all instances for (name, status, disks_active, snodes, pnode_group_uuid, snodes_group_uuid) in raw_instances: if snodes and set([pnode_group_uuid]) != set(snodes_group_uuid): logging.error( "Ignoring split instance '%s', primary group %s, secondary" " groups %s", name, pnode_group_uuid, utils.CommaJoin(snodes_group_uuid)) else: instances.append(Instance(name, status, disks_active, snodes)) for node in snodes: secondaries.setdefault(node, set()).add(name) # Load all nodes nodes = [ Node(name, bootid, offline, secondaries.get(name, set())) for (name, bootid, offline) in raw_nodes ] return (dict((node.name, node) for node in nodes), dict((inst.name, inst) for inst in instances))
class _OpTestVerifyErrors(opcodes.OpCode): OP_PARAMS = [ ("debug_simulate_errors", False, ht.TBool, ""), ("error_codes", False, ht.TBool, ""), ("ignore_errors", [], ht.TListOf(ht.TElemOf(constants.CV_ALL_ECODES_STRINGS)), "") ]
def testListOf(self): fn = ht.TListOf(ht.TNonEmptyString) self.assertTrue(fn([])) self.assertTrue(fn(["x"])) self.assertTrue(fn(["Hello", "World"])) self.assertFalse(fn(None)) self.assertFalse(fn(False)) self.assertFalse(fn(range(3))) self.assertFalse(fn(["x", None]))
def testVerificationFails(self): self.assertRaises(errors.ParseError, serializer.LoadAndVerifyJson, "{}", lambda _: False) verify_fn = ht.TListOf(ht.TNonEmptyString) try: serializer.LoadAndVerifyJson("{}", verify_fn) except errors.ParseError, err: self.assertTrue(str(err).endswith(str(verify_fn)))
def BuildJobDepCheck(relative): """Builds check for job dependencies (L{DEPEND_ATTR}). @type relative: bool @param relative: Whether to accept relative job IDs (negative) @rtype: callable """ if relative: job_id = ht.TOr(ht.TJobId, ht.TRelativeJobId) else: job_id = ht.TJobId job_dep = \ ht.TAnd(ht.TOr(ht.TListOf(ht.TAny), ht.TTuple), ht.TIsLength(2), ht.TItems([job_id, ht.TListOf(ht.TElemOf(constants.JOBS_FINALIZED))])) return ht.TMaybe(ht.TListOf(job_dep))
class IAReqMultiInstanceAlloc(IARequestBase): """An multi instance allocation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_MULTI_ALLOC REQ_PARAMS = [ ("instances", ht.TListOf(ht.TInstanceOf(IAReqInstanceAlloc))), ] _MASUCCESS = \ ht.TListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString), ]))) _MAFAILED = ht.TListOf(ht.TNonEmptyString) REQ_RESULT = ht.TAnd(ht.TList, ht.TIsLength(2), ht.TItems([_MASUCCESS, _MAFAILED])) def GetRequest(self, cfg): return { "instances": [iareq.GetRequest(cfg) for iareq in self.instances], }
def GetPostOpInput(self): """Replaces disks on an instance. """ static = { "instance_name": self.items[0], } if self.request_body: data = self.request_body elif self.queryargs: # Legacy interface, do not modify/extend data = { "remote_node": self._checkStringVariable("remote_node", default=None), "mode": self._checkStringVariable("mode", default=None), "disks": self._checkStringVariable("disks", default=None), "iallocator": self._checkStringVariable("iallocator", default=None), } else: data = {} # Parse disks try: raw_disks = data.pop("disks") except KeyError: pass else: if raw_disks: if ht.TListOf(ht.TInt)(raw_disks): # pylint: disable=E1102 data["disks"] = raw_disks else: # Backwards compatibility for strings of the format "1, 2, 3" try: data["disks"] = [ int(part) for part in raw_disks.split(",") ] except (TypeError, ValueError), err: raise http.HttpBadRequest( "Invalid disk index passed: %s" % err)
""" from ganeti import constants from ganeti import errors from ganeti import ht _FOURCC_LEN = 4 #: Items in the individual rows of the NodeDrbd query _HTNodeDrbdItems = [ ht.TString, ht.TInt, ht.TString, ht.TString, ht.TString, ht.TString ] #: Type for the (top-level) result of NodeDrbd query HTNodeDrbd = ht.TListOf( ht.TAnd(ht.TList, ht.TIsLength(len(_HTNodeDrbdItems)), ht.TItems(_HTNodeDrbdItems))) def PackMagic(payload): """Prepend the confd magic fourcc to a payload. """ return "".join([constants.CONFD_MAGIC_FOURCC, payload]) def UnpackMagic(payload): """Unpack and check the confd magic fourcc from a payload. """ if len(payload) < _FOURCC_LEN:
# TODO: Test dependencies on errors jobs = [ [opcodes.OpTestDelay(duration=1)], [opcodes.OpTestDelay(duration=1, depends=[(-1, [])])], [opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])])], [opcodes.OpTestDelay(duration=1, depends=[])], [opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])])], ] # Function for checking result check_fn = ht.TListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TBool, ht.TOr(ht.TNonEmptyString, ht.TJobId)]))) cl = cli.GetClient() result = cl.SubmitManyJobs(jobs) if not check_fn(result): raise errors.OpExecError("Job submission doesn't match %s: %s" % (check_fn, result)) # Wait for jobs to finish jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result: jex.AddJobId(None, status, job_id) job_results = jex.GetResults()
from ganeti import utils from ganeti import ht from ganeti import ssh from ganeti import pathutils from ganeti.tools import common _DATA_CHECK = ht.TStrictDict( False, True, { constants.SSHS_CLUSTER_NAME: ht.TNonEmptyString, constants.SSHS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.SSHS_SSH_PUBLIC_KEYS: ht.TItems([ ht.TElemOf(constants.SSHS_ACTIONS), ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString)) ]), constants.SSHS_SSH_AUTHORIZED_KEYS: ht.TItems([ ht.TElemOf(constants.SSHS_ACTIONS), ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString)) ]), constants.SSHS_GENERATE: ht.TDictOf(ht.TNonEmptyString, ht.TString), }) class SshUpdateError(errors.GenericError): """Local class for reporting errors. """
from ganeti import pathutils from ganeti import utils from ganeti import ht from ganeti import ssh from ganeti.tools import common _SSH_KEY_LIST_ITEM = \ ht.TAnd(ht.TIsLength(3), ht.TItems([ ht.TSshKeyType, ht.Comment("public")(ht.TNonEmptyString), ht.Comment("private")(ht.TNonEmptyString), ])) _SSH_KEY_LIST = ht.TListOf(_SSH_KEY_LIST_ITEM) _DATA_CHECK = ht.TStrictDict( False, True, { constants.SSHS_CLUSTER_NAME: ht.TNonEmptyString, constants.SSHS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.SSHS_SSH_HOST_KEY: _SSH_KEY_LIST, constants.SSHS_SSH_ROOT_KEY: _SSH_KEY_LIST, constants.SSHS_SSH_AUTHORIZED_KEYS: ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString)), constants.SSHS_SSH_KEY_TYPE: ht.TSshKeyType,
def _GetGroupData(qcl, uuid): """Retrieves instances and nodes per node group. """ locks = qcl.Query(constants.QR_LOCK, ["name", "mode"], None) prefix = "instance/" prefix_len = len(prefix) locked_instances = set() for [[_, name], [_, lock]] in locks.data: if name.startswith(prefix) and lock: locked_instances.add(name[prefix_len:]) queries = [ (constants.QR_INSTANCE, [ "name", "status", "admin_state", "admin_state_source", "disks_active", "snodes", "pnode.group.uuid", "snodes.group.uuid", "disk_template" ], [qlang.OP_EQUAL, "pnode.group.uuid", uuid]), (constants.QR_NODE, ["name", "bootid", "offline"], [qlang.OP_EQUAL, "group.uuid", uuid]), ] results_data = [ qcl.Query(what, field, qfilter).data for (what, field, qfilter) in queries ] # Ensure results are tuples with two values assert compat.all( ht.TListOf(ht.TListOf(ht.TIsLength(2)))(d) for d in results_data) # Extract values ignoring result status (raw_instances, raw_nodes) = [[[v[1] for v in values] for values in res] for res in results_data] secondaries = {} instances = [] # Load all instances for (name, status, config_state, config_state_source, disks_active, snodes, pnode_group_uuid, snodes_group_uuid, disk_template) in raw_instances: if snodes and set([pnode_group_uuid]) != set(snodes_group_uuid): logging.error( "Ignoring split instance '%s', primary group %s, secondary" " groups %s", name, pnode_group_uuid, utils.CommaJoin(snodes_group_uuid)) else: instances.append( Instance(name, status, config_state, config_state_source, disks_active, snodes, disk_template)) for node in snodes: secondaries.setdefault(node, set()).add(name) # Load all nodes nodes = [ Node(name, bootid, offline, secondaries.get(name, set())) for (name, bootid, offline) in raw_nodes ] return (dict((node.name, node) for node in nodes), dict((inst.name, inst) for inst in instances), locked_instances)
def _TestJobDependency(opts): """Tests job dependencies. """ ToStdout("Testing job dependencies") try: cl = cli.GetClient() SubmitOpCode(opcodes.OpTestDelay(duration=0, depends=[(-1, None)]), cl=cl) except errors.GenericError as err: if opts.debug: ToStdout("Ignoring error for 'wrong dependencies' test: %s", err) else: raise errors.OpExecError("Submitting plain opcode with relative job ID" " did not fail as expected") # TODO: Test dependencies on errors jobs = [ [opcodes.OpTestDelay(duration=1)], [opcodes.OpTestDelay(duration=1, depends=[(-1, [])])], [ opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])]) ], [opcodes.OpTestDelay(duration=1, depends=[])], [ opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])]) ], ] # Function for checking result check_fn = ht.TListOf( ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TBool, ht.TOr(ht.TNonEmptyString, ht.TJobId)]))) cl = cli.GetClient() result = cl.SubmitManyJobs(jobs) if not check_fn(result): raise errors.OpExecError("Job submission doesn't match %s: %s" % (check_fn, result)) # Wait for jobs to finish jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result: jex.AddJobId(None, status, job_id) job_results = jex.GetResults() if not compat.all(row[0] for row in job_results): raise errors.OpExecError( "At least one of the submitted jobs failed: %s" % job_results) # Get details about jobs data = cl.QueryJobs([job_id for (_, job_id) in result], ["id", "opexec", "ops"]) data_job_id = [job_id for (job_id, _, _) in data] data_opexec = [opexec for (_, opexec, _) in data] data_op = [[opcodes.OpCode.LoadOpCode(op) for op in ops] for (_, _, ops) in data] assert compat.all(not op.depends or len(op.depends) == 1 for ops in data_op for op in ops) # Check resolved job IDs in dependencies for (job_idx, res_jobdep) in [(1, data_job_id[0]), (2, data_job_id[0]), (4, data_job_id[2])]: if data_op[job_idx][0].depends[0][0] != res_jobdep: raise errors.OpExecError( "Job %s's opcode doesn't depend on correct job" " ID (%s)" % (job_idx, res_jobdep)) # Check execution order if not (data_opexec[0] <= data_opexec[1] and data_opexec[0] <= data_opexec[2] and data_opexec[2] <= data_opexec[4]): raise errors.OpExecError("Jobs did not run in correct order: %s" % data) assert len(jobs) == 5 and compat.all(len(ops) == 1 for ops in jobs) ToStdout("Job dependency tests were successful")
class IAReqInstanceAlloc(IARequestBase): """An instance allocation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_ALLOC REQ_PARAMS = [ _INST_NAME, ("memory", ht.TNonNegativeInt), ("spindle_use", ht.TNonNegativeInt), ("disks", ht.TListOf(ht.TDict)), ("disk_template", ht.TString), ("group_name", ht.TMaybe(ht.TNonEmptyString)), ("os", ht.TString), ("tags", _STRING_LIST), ("nics", ht.TListOf(ht.TDict)), ("vcpus", ht.TInt), ("hypervisor", ht.TString), ("node_whitelist", ht.TMaybeListOf(ht.TNonEmptyString)), ] REQ_RESULT = ht.TList def RequiredNodes(self): """Calculates the required nodes based on the disk_template. """ if self.disk_template in constants.DTS_INT_MIRROR: return 2 else: return 1 def GetRequest(self, cfg): """Requests a new instance. The checks for the completeness of the opcode must have already been done. """ for d in self.disks: d[constants.IDISK_TYPE] = self.disk_template disk_space = gmi.ComputeDiskSize(self.disks) return { "name": self.name, "disk_template": self.disk_template, "group_name": self.group_name, "tags": self.tags, "os": self.os, "vcpus": self.vcpus, "memory": self.memory, "spindle_use": self.spindle_use, "disks": self.disks, "disk_space_total": disk_space, "nics": self.nics, "required_nodes": self.RequiredNodes(), "hypervisor": self.hypervisor, } def ValidateResult(self, ia, result): """Validates an single instance allocation request. """ IARequestBase.ValidateResult(self, ia, result) if ia.success and len(result) != self.RequiredNodes(): raise errors.ResultValidationError( "iallocator returned invalid number" " of nodes (%s), required %s" % (len(result), self.RequiredNodes()))
import logging from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import outils from ganeti import opcodes from ganeti import serializer from ganeti import utils import ganeti.rpc.node as rpc import ganeti.masterd.instance as gmi _STRING_LIST = ht.TListOf(ht.TString) _JOB_LIST = ht.TListOf( ht.TListOf( ht.TStrictDict( True, False, { # pylint: disable=E1101 # Class '...' has no 'OP_ID' member "OP_ID": ht.TElemOf([ opcodes.OpInstanceFailover.OP_ID, opcodes.OpInstanceMigrate.OP_ID, opcodes.OpInstanceReplaceDisks.OP_ID ]), })))
from ganeti import utils from ganeti import serializer from ganeti import ht from ganeti import ssh from ganeti import ssconf _SSH_KEY_LIST_ITEM = \ ht.TAnd(ht.TIsLength(3), ht.TItems([ ht.TElemOf(constants.SSHK_ALL), ht.Comment("public")(ht.TNonEmptyString), ht.Comment("private")(ht.TNonEmptyString), ])) _SSH_KEY_LIST = ht.TListOf(_SSH_KEY_LIST_ITEM) _DATA_CHECK = ht.TStrictDict( False, True, { constants.SSHS_CLUSTER_NAME: ht.TNonEmptyString, constants.SSHS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.SSHS_SSH_HOST_KEY: _SSH_KEY_LIST, constants.SSHS_SSH_ROOT_KEY: _SSH_KEY_LIST, }) class JoinError(errors.GenericError): """Local class for reporting errors. """