def set_sriov_vf_vm(zkhandler, vm_uuid, node, vf, vf_macaddr, vf_type):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
return False
# Verify VF is valid
vf_information = getSRIOVVFInformation(zkhandler, node, vf)
if not vf_information:
return False
update_list = [
(("node.sriov.vf", node, "sriov_vf.used", vf), "True"),
(("node.sriov.vf", node, "sriov_vf.used_by", vf), vm_uuid),
(("node.sriov.vf", node, "sriov_vf.mac", vf), vf_macaddr),
]
# Hostdev type SR-IOV prevents the guest from live migrating
if vf_type == "hostdev":
update_list.append(
(("domain.meta.migrate_method", vm_uuid), "shutdown"))
zkhandler.write(update_list)
return True
def secondary_node(zkhandler, node):
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node
)
# Ensure node is a coordinator
daemon_mode = zkhandler.read(("node.mode", node))
if daemon_mode == "hypervisor":
return (
False,
'ERROR: Cannot change router mode on non-coordinator node "{}"'.format(
node
),
)
# Ensure node is in run daemonstate
daemon_state = zkhandler.read(("node.state.daemon", node))
if daemon_state != "run":
return False, 'ERROR: Node "{}" is not active'.format(node)
# Get current state
current_state = zkhandler.read(("node.state.router", node))
if current_state == "secondary":
return True, 'Node "{}" is already in secondary router mode.'.format(node)
retmsg = "Setting node {} in secondary router mode.".format(node)
zkhandler.write([("base.config.primary_node", "none")])
return True, retmsg
def add_osd_db_vg(zkhandler, node, device):
# Verify the target node exists
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node)
# Tell the cluster to create a new OSD for the host
add_osd_db_vg_string = "db_vg_add {},{}".format(node, device)
zkhandler.write([("base.cmd.ceph", add_osd_db_vg_string)])
# Wait 1/2 second for the cluster to get the message and start working
time.sleep(0.5)
# Acquire a read lock, so we get the return exclusively
with zkhandler.readlock("base.cmd.ceph"):
try:
result = zkhandler.read("base.cmd.ceph").split()[0]
if result == "success-db_vg_add":
message = 'Created new OSD database VG at "{}" on node "{}".'.format(
device, node)
success = True
else:
message = "ERROR: Failed to create new OSD database VG; check node logs for details."
success = False
except Exception:
message = "ERROR: Command ignored by node."
success = False
# Acquire a write lock to ensure things go smoothly
with zkhandler.writelock("base.cmd.ceph"):
time.sleep(0.5)
zkhandler.write([("base.cmd.ceph", "")])
return success, message
def get_info_sriov_vf(zkhandler, node, vf):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(node)
vf_information = getSRIOVVFInformation(zkhandler, node, vf)
if not vf_information:
return False, 'ERROR: Could not find SR-IOV VF "{}" on node "{}"'.format(
vf, node)
return True, vf_information
def get_info(zkhandler, node):
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node
)
# Get information about node in a pretty format
node_information = getNodeInformation(zkhandler, node)
if not node_information:
return False, 'ERROR: Could not get information about node "{}".'.format(node)
return True, node_information
def add_osd(zkhandler,
node,
device,
weight,
ext_db_flag=False,
ext_db_ratio=0.05):
# Verify the target node exists
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node)
# Verify target block device isn't in use
block_osd = verifyOSDBlock(zkhandler, node, device)
if block_osd:
return (
False,
'ERROR: Block device "{}" on node "{}" is used by OSD "{}"'.format(
device, node, block_osd),
)
# Tell the cluster to create a new OSD for the host
add_osd_string = "osd_add {},{},{},{},{}".format(node, device, weight,
ext_db_flag, ext_db_ratio)
zkhandler.write([("base.cmd.ceph", add_osd_string)])
# Wait 1/2 second for the cluster to get the message and start working
time.sleep(0.5)
# Acquire a read lock, so we get the return exclusively
with zkhandler.readlock("base.cmd.ceph"):
try:
result = zkhandler.read("base.cmd.ceph").split()[0]
if result == "success-osd_add":
message = 'Created new OSD with block device "{}" on node "{}".'.format(
device, node)
success = True
else:
message = (
"ERROR: Failed to create new OSD; check node logs for details."
)
success = False
except Exception:
message = "ERROR: Command ignored by node."
success = False
# Acquire a write lock to ensure things go smoothly
with zkhandler.writelock("base.cmd.ceph"):
time.sleep(0.5)
zkhandler.write([("base.cmd.ceph", "")])
return success, message
def get_node_log(zkhandler, node, lines=2000):
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node
)
# Get the data from ZK
node_log = zkhandler.read(("logs.messages", node))
if node_log is None:
return True, ""
# Shrink the log buffer to length lines
shrunk_log = node_log.split("\n")[-lines:]
loglines = "\n".join(shrunk_log)
return True, loglines
def get_list_sriov_vf(zkhandler, node, pf=None):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(node)
vf_list = list()
vf_phy_list = common.sortInterfaceNames(
zkhandler.children(("node.sriov.vf", node)))
for phy in vf_phy_list:
retcode, vf_information = get_info_sriov_vf(zkhandler, node, phy)
if retcode:
if pf is not None:
if vf_information["pf"] == pf:
vf_list.append(vf_information)
else:
vf_list.append(vf_information)
return True, vf_list
def ready_node(zkhandler, node, wait=False):
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node
)
if zkhandler.read(("node.state.domain", node)) == "ready":
return True, "Hypervisor {} is already ready.".format(node)
retmsg = "Restoring hypervisor {} to active service.".format(node)
# Add the new domain to Zookeeper
zkhandler.write([(("node.state.domain", node), "unflush")])
if wait:
while zkhandler.read(("node.state.domain", node)) == "unflush":
time.sleep(1)
retmsg = "Restored hypervisor {} to active service.".format(node)
return True, retmsg
def flush_node(zkhandler, node, wait=False):
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'ERROR: No node named "{}" is present in the cluster.'.format(
node
)
if zkhandler.read(("node.state.domain", node)) == "flushed":
return True, "Hypervisor {} is already flushed.".format(node)
retmsg = "Flushing hypervisor {} of running VMs.".format(node)
# Add the new domain to Zookeeper
zkhandler.write([(("node.state.domain", node), "flush")])
if wait:
while zkhandler.read(("node.state.domain", node)) == "flush":
time.sleep(1)
retmsg = "Flushed hypervisor {} of running VMs.".format(node)
return True, retmsg
def unset_sriov_vf_vm(zkhandler, node, vf):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
return False
# Verify VF is valid
vf_information = getSRIOVVFInformation(zkhandler, node, vf)
if not vf_information:
return False
update_list = [
(("node.sriov.vf", node, "sriov_vf.used", vf), "False"),
(("node.sriov.vf", node, "sriov_vf.used_by", vf), ""),
(
("node.sriov.vf", node, "sriov_vf.mac", vf),
zkhandler.read(("node.sriov.vf", node, "sriov_vf.phy_mac", vf)),
),
]
zkhandler.write(update_list)
return True
def set_sriov_vf_config(
zkhandler,
node,
vf,
vlan_id=None,
vlan_qos=None,
tx_rate_min=None,
tx_rate_max=None,
link_state=None,
spoof_check=None,
trust=None,
query_rss=None,
):
# Verify node is valid
valid_node = common.verifyNode(zkhandler, node)
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(node)
# Verify VF is valid
vf_information = getSRIOVVFInformation(zkhandler, node, vf)
if not vf_information:
return False, 'ERROR: Could not find SR-IOV VF "{}" on node "{}".'.format(
vf, node)
update_list = list()
if vlan_id is not None:
update_list.append(
(("node.sriov.vf", node, "sriov_vf.config.vlan_id", vf), vlan_id))
if vlan_qos is not None:
update_list.append((("node.sriov.vf", node, "sriov_vf.config.vlan_qos",
vf), vlan_qos))
if tx_rate_min is not None:
update_list.append((("node.sriov.vf", node,
"sriov_vf.config.tx_rate_min", vf), tx_rate_min))
if tx_rate_max is not None:
update_list.append((("node.sriov.vf", node,
"sriov_vf.config.tx_rate_max", vf), tx_rate_max))
if link_state is not None:
update_list.append((("node.sriov.vf", node,
"sriov_vf.config.link_state", vf), link_state))
if spoof_check is not None:
update_list.append((("node.sriov.vf", node,
"sriov_vf.config.spoof_check", vf), spoof_check))
if trust is not None:
update_list.append(
(("node.sriov.vf", node, "sriov_vf.config.trust", vf), trust))
if query_rss is not None:
update_list.append((("node.sriov.vf", node,
"sriov_vf.config.query_rss", vf), query_rss))
if len(update_list) < 1:
return False, "ERROR: No changes to apply."
result = zkhandler.write(update_list)
if result:
return (
True,
'Successfully modified configuration of SR-IOV VF "{}" on node "{}".'
.format(vf, node),
)
else:
return (
False,
'Failed to modify configuration of SR-IOV VF "{}" on node "{}".'.
format(vf, node),
)
def move_vm(zkhandler, domain, target_node, wait=False, force_live=False):
# Validate that VM exists in cluster
dom_uuid = getDomainUUID(zkhandler, domain)
if not dom_uuid:
return False, 'ERROR: Could not find VM "{}" in the cluster!'.format(domain)
# Get state and verify we're OK to proceed
current_state = zkhandler.read(("domain.state", dom_uuid))
if current_state != "start":
# If the current state isn't start, preserve it; we're not doing live migration
target_state = current_state
else:
if force_live:
target_state = "migrate-live"
else:
target_state = "migrate"
current_node = zkhandler.read(("domain.node", dom_uuid))
if not target_node:
target_node = common.findTargetNode(zkhandler, dom_uuid)
else:
# Verify node is valid
valid_node = common.verifyNode(zkhandler, target_node)
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(target_node)
# Check if node is within the limit
node_limit = zkhandler.read(("domain.meta.node_limit", dom_uuid))
if node_limit and target_node not in node_limit.split(","):
return (
False,
'ERROR: Specified node "{}" is not in the allowed list of nodes for VM "{}".'.format(
target_node, domain
),
)
# Verify if node is current node
if target_node == current_node:
last_node = zkhandler.read(("domain.last_node", dom_uuid))
if last_node:
zkhandler.write([(("domain.last_node", dom_uuid), "")])
return True, 'Making temporary migration permanent for VM "{}".'.format(
domain
)
return False, 'ERROR: VM "{}" is already running on node "{}".'.format(
domain, current_node
)
if not target_node:
return (
False,
'ERROR: Could not find a valid migration target for VM "{}".'.format(
domain
),
)
retmsg = 'Permanently migrating VM "{}" to node "{}".'.format(domain, target_node)
lock = zkhandler.exclusivelock(("domain.state", dom_uuid))
with lock:
zkhandler.write(
[
(("domain.state", dom_uuid), target_state),
(("domain.node", dom_uuid), target_node),
(("domain.last_node", dom_uuid), ""),
]
)
# Wait for 1/2 second for migration to start
time.sleep(0.5)
# Update any SR-IOV NICs
update_vm_sriov_nics(zkhandler, dom_uuid, current_node, target_node)
if wait:
while zkhandler.read(("domain.state", dom_uuid)) == target_state:
time.sleep(0.5)
retmsg = 'Permanently migrated VM "{}" to node "{}"'.format(domain, target_node)
return True, retmsg
def define_vm(
zkhandler,
config_data,
target_node,
node_limit,
node_selector,
node_autostart,
migration_method=None,
profile=None,
tags=[],
initial_state="stop",
):
# Parse the XML data
try:
parsed_xml = lxml.objectify.fromstring(config_data)
except Exception:
return False, "ERROR: Failed to parse XML data."
dom_uuid = parsed_xml.uuid.text
dom_name = parsed_xml.name.text
# Ensure that the UUID and name are unique
if searchClusterByUUID(zkhandler, dom_uuid) or searchClusterByName(
zkhandler, dom_name
):
return (
False,
'ERROR: Specified VM "{}" or UUID "{}" matches an existing VM on the cluster'.format(
dom_name, dom_uuid
),
)
if not target_node:
target_node = common.findTargetNode(zkhandler, dom_uuid)
else:
# Verify node is valid
valid_node = common.verifyNode(zkhandler, target_node)
if not valid_node:
return False, 'ERROR: Specified node "{}" is invalid.'.format(target_node)
# Validate the new RAM against the current active node
node_total_memory = int(zkhandler.read(("node.memory.total", target_node)))
if int(parsed_xml.memory.text) >= node_total_memory:
return (
False,
'ERROR: VM configuration specifies more memory ({} MiB) than node "{}" has available ({} MiB).'.format(
parsed_xml.memory.text, target_node, node_total_memory
),
)
# Validate the number of vCPUs against the current active node
node_total_cpus = int(zkhandler.read(("node.data.static", target_node)).split()[0])
if (node_total_cpus - 2) <= int(parsed_xml.vcpu.text):
return (
False,
'ERROR: VM configuration specifies more vCPUs ({}) than node "{}" has available ({} minus 2).'.format(
parsed_xml.vcpu.text, target_node, node_total_cpus
),
)
# If a SR-IOV network device is being added, set its used state
dnetworks = common.getDomainNetworks(parsed_xml, {})
for network in dnetworks:
if network["type"] in ["direct", "hostdev"]:
dom_node = zkhandler.read(("domain.node", dom_uuid))
# Check if the network is already in use
is_used = zkhandler.read(
("node.sriov.vf", dom_node, "sriov_vf.used", network["source"])
)
if is_used == "True":
used_by_name = searchClusterByUUID(
zkhandler,
zkhandler.read(
(
"node.sriov.vf",
dom_node,
"sriov_vf.used_by",
network["source"],
)
),
)
return (
False,
'ERROR: Attempted to use SR-IOV network "{}" which is already used by VM "{}" on node "{}".'.format(
network["source"], used_by_name, dom_node
),
)
# We must update the "used" section
set_sriov_vf_vm(
zkhandler,
dom_uuid,
dom_node,
network["source"],
network["mac"],
network["type"],
)
# Obtain the RBD disk list using the common functions
ddisks = common.getDomainDisks(parsed_xml, {})
rbd_list = []
for disk in ddisks:
if disk["type"] == "rbd":
rbd_list.append(disk["name"])
# Join the limit
if isinstance(node_limit, list) and node_limit:
formatted_node_limit = ",".join(node_limit)
else:
formatted_node_limit = ""
# Join the RBD list
if isinstance(rbd_list, list) and rbd_list:
formatted_rbd_list = ",".join(rbd_list)
else:
formatted_rbd_list = ""
# Add the new domain to Zookeeper
zkhandler.write(
[
(("domain", dom_uuid), dom_name),
(("domain.xml", dom_uuid), config_data),
(("domain.state", dom_uuid), initial_state),
(("domain.profile", dom_uuid), profile),
(("domain.stats", dom_uuid), ""),
(("domain.node", dom_uuid), target_node),
(("domain.last_node", dom_uuid), ""),
(("domain.failed_reason", dom_uuid), ""),
(("domain.storage.volumes", dom_uuid), formatted_rbd_list),
(("domain.console.log", dom_uuid), ""),
(("domain.console.vnc", dom_uuid), ""),
(("domain.meta.autostart", dom_uuid), node_autostart),
(("domain.meta.migrate_method", dom_uuid), migration_method),
(("domain.meta.node_limit", dom_uuid), formatted_node_limit),
(("domain.meta.node_selector", dom_uuid), node_selector),
(("domain.meta.tags", dom_uuid), ""),
(("domain.migrate.sync_lock", dom_uuid), ""),
]
)
for tag in tags:
tag_name = tag["name"]
zkhandler.write(
[
(("domain.meta.tags", dom_uuid, "tag.name", tag_name), tag["name"]),
(("domain.meta.tags", dom_uuid, "tag.type", tag_name), tag["type"]),
(
("domain.meta.tags", dom_uuid, "tag.protected", tag_name),
tag["protected"],
),
]
)
return True, 'Added new VM with Name "{}" and UUID "{}" to database.'.format(
dom_name, dom_uuid
)
def get_list(zkhandler, node, state, tag, limit, is_fuzzy=True, negate=False):
if node:
# Verify node is valid
if not common.verifyNode(zkhandler, node):
return False, 'Specified node "{}" is invalid.'.format(node)
if state:
valid_states = [
"start",
"restart",
"shutdown",
"stop",
"disable",
"fail",
"migrate",
"unmigrate",
"provision",
]
if state not in valid_states:
return False, 'VM state "{}" is not valid.'.format(state)
full_vm_list = zkhandler.children("base.domain")
# Set our limit to a sensible regex
if limit:
# Check if the limit is a UUID
is_limit_uuid = False
try:
uuid_obj = UUID(limit, version=4)
limit = str(uuid_obj)
is_limit_uuid = True
except ValueError:
pass
if is_fuzzy and not is_limit_uuid:
try:
# Implcitly assume fuzzy limits
if not re.match(r"\^.*", limit):
limit = ".*" + limit
if not re.match(r".*\$", limit):
limit = limit + ".*"
except Exception as e:
return False, "Regex Error: {}".format(e)
get_vm_info = dict()
for vm in full_vm_list:
name = zkhandler.read(("domain", vm))
is_limit_match = False
is_tag_match = False
is_node_match = False
is_state_match = False
# Check on limit
if limit:
# Try to match the limit against the UUID (if applicable) and name
try:
if is_limit_uuid and re.fullmatch(limit, vm):
is_limit_match = True
if re.fullmatch(limit, name):
is_limit_match = True
except Exception as e:
return False, "Regex Error: {}".format(e)
else:
is_limit_match = True
if tag:
vm_tags = zkhandler.children(("domain.meta.tags", vm))
if negate and tag not in vm_tags:
is_tag_match = True
if not negate and tag in vm_tags:
is_tag_match = True
else:
is_tag_match = True
# Check on node
if node:
vm_node = zkhandler.read(("domain.node", vm))
if negate and vm_node != node:
is_node_match = True
if not negate and vm_node == node:
is_node_match = True
else:
is_node_match = True
# Check on state
if state:
vm_state = zkhandler.read(("domain.state", vm))
if negate and vm_state != state:
is_state_match = True
if not negate and vm_state == state:
is_state_match = True
else:
is_state_match = True
get_vm_info[vm] = (
True
if is_limit_match and is_tag_match and is_node_match and is_state_match
else False
)
# Obtain our VM data in a thread pool
# This helps parallelize the numerous Zookeeper calls a bit, within the bounds of the GIL, and
# should help prevent this task from becoming absurdly slow with very large numbers of VMs.
# The max_workers is capped at 32 to avoid creating an absurd number of threads especially if
# the list gets called multiple times simultaneously by the API, but still provides a noticeable
# speedup.
vm_execute_list = [vm for vm in full_vm_list if get_vm_info[vm]]
vm_data_list = list()
with ThreadPoolExecutor(max_workers=32, thread_name_prefix="vm_list") as executor:
futures = []
for vm_uuid in vm_execute_list:
futures.append(
executor.submit(common.getInformationFromXML, zkhandler, vm_uuid)
)
for future in futures:
try:
vm_data_list.append(future.result())
except Exception:
pass
return True, vm_data_list