def get_network(network_id, for_update=True):
"""Get a Network object by its ID.
@type network_id: int or string
@param network_id: The networks DB id or the Ganeti name
"""
try:
network_id = int(network_id)
except (ValueError, TypeError):
try:
network_id = id_from_network_name(network_id)
except Network.InvalidBackendIdError:
raise CommandError("Invalid network ID: %s" % network_id)
networks = Network.objects
if for_update:
networks = networks.select_for_update()
try:
return networks.get(id=network_id)
except Network.DoesNotExist:
raise CommandError("Network with ID %s not found in DB."
" Use snf-manage network-list to find out"
" available network IDs." % network_id)
def get_network(network_id, for_update=True):
"""Get a Network object by its ID.
@type network_id: int or string
@param network_id: The networks DB id or the Ganeti name
"""
try:
network_id = int(network_id)
except (ValueError, TypeError):
try:
network_id = id_from_network_name(network_id)
except Network.InvalidBackendIdError:
raise CommandError("Invalid network ID: %s" % network_id)
networks = Network.objects
if for_update:
networks = networks.select_for_update()
try:
return networks.get(id=network_id)
except Network.DoesNotExist:
raise CommandError("Network with ID %s not found in DB."
" Use snf-manage network-list to find out"
" available network IDs." % network_id)
def process_ganeti_nics(ganeti_nics):
"""Process NIC dict from ganeti hooks."""
new_nics = []
for i, new_nic in enumerate(ganeti_nics):
network = new_nic.get('network', '')
n = str(network)
pk = utils.id_from_network_name(n)
net = Network.objects.get(pk=pk)
# Get the new nic info
mac = new_nic.get('mac', '')
ipv4 = new_nic.get('ip', '')
if net.subnet6:
ipv6 = mac2eui64(mac, net.subnet6)
else:
ipv6 = ''
firewall = new_nic.get('firewall', '')
firewall_profile = _reverse_tags.get(firewall, '')
if not firewall_profile and net.public:
firewall_profile = settings.DEFAULT_FIREWALL_PROFILE
nic = {
'index': i,
'network': net,
'mac': mac,
'ipv4': ipv4,
'ipv6': ipv6,
'firewall_profile': firewall_profile,
'state': 'ACTIVE'}
new_nics.append(nic)
return new_nics
def get_networks_from_ganeti(backend):
prefix = settings.BACKEND_PREFIX_ID + 'net-'
networks = {}
with pooled_rapi_client(backend) as c:
for net in c.GetNetworks(bulk=True):
if net['name'].startswith(prefix):
id = utils.id_from_network_name(net['name'])
networks[id] = net
return networks
def get_networks_from_ganeti(backend):
prefix = settings.BACKEND_PREFIX_ID + 'net-'
networks = {}
with pooled_rapi_client(backend) as c:
for net in c.GetNetworks(bulk=True):
if net['name'].startswith(prefix):
id = utils.id_from_network_name(net['name'])
networks[id] = net
return networks
def wrapper(msg):
try:
network_id = utils.id_from_network_name(msg["network"])
network = Network.objects.select_for_update().get(id=network_id)
backend = Backend.objects.get(clustername=msg["cluster"])
bnet, new = BackendNetwork.objects.get_or_create(network=network, backend=backend)
if new:
log.info("Created missing BackendNetwork %s", bnet)
func(bnet, msg)
except Network.InvalidBackendIdError:
log.debug("Ignoring msg for unknown network %s.", msg["network"])
except Network.DoesNotExist:
log.error("Network %s not found in DB.", msg["network"])
except Backend.DoesNotExist:
log.error("Backend %s not found in DB.", msg["cluster"])
except BackendNetwork.DoesNotExist:
log.error("Network %s on backend %s not found in DB.", msg["network"], msg["cluster"])
def wrapper(msg):
try:
network_id = utils.id_from_network_name(msg["network"])
network = Network.objects.select_for_update().get(id=network_id)
backend = Backend.objects.get(clustername=msg['cluster'])
bnet, new = BackendNetwork.objects.get_or_create(network=network,
backend=backend)
if new:
log.info("Created missing BackendNetwork %s", bnet)
func(bnet, msg)
except Network.InvalidBackendIdError:
log.debug("Ignoring msg for unknown network %s.", msg['network'])
except Network.DoesNotExist:
log.error("Network %s not found in DB.", msg['network'])
except Backend.DoesNotExist:
log.error("Backend %s not found in DB.", msg['cluster'])
except BackendNetwork.DoesNotExist:
log.error("Network %s on backend %s not found in DB.",
msg['network'], msg['cluster'])
def get_resource(name, value, for_update=False):
"""Get object from DB based by it's ID
Helper function for getting an object from DB by it's DB and raising
appropriate command line errors if the object does not exist or the
ID is invalid.
"""
objects = RESOURCE_MAP[name]
if name == "floating-ip":
capital_name = "Floating IP"
else:
capital_name = name.capitalize()
PREFIXED_RESOURCES = ["server", "network", "port", "volume"]
if isinstance(value, basestring) and name in PREFIXED_RESOURCES:
if value.startswith(settings.BACKEND_PREFIX_ID):
try:
if name == "server":
value = id_from_instance_name(value)
elif name == "network":
value = id_from_network_name(value)
elif name == "port":
value = id_from_nic_name(value)
elif name == "volume":
value = id_from_disk_name(value)
except ValueError:
raise CommandError("Invalid {} ID: {}".format(capital_name,
value))
if for_update:
objects = objects.select_for_update()
try:
return objects.get(id=value)
except ObjectDoesNotExist:
msg = ("{0} with ID {1} does not exist. Use {2}-list to find out"
" available {2} IDs.")
raise CommandError(msg.format(capital_name, value, name))
except (ValueError, TypeError):
raise CommandError("Invalid {} ID: {}".format(capital_name, value))
def get_resource(name, value, for_update=False):
"""Get object from DB based by it's ID
Helper function for getting an object from DB by it's DB and raising
appropriate command line errors if the object does not exist or the
ID is invalid.
"""
objects = RESOURCE_MAP[name]
if name == "floating-ip":
capital_name = "Floating IP"
else:
capital_name = name.capitalize()
PREFIXED_RESOURCES = ["server", "network", "port", "volume"]
if isinstance(value, basestring) and name in PREFIXED_RESOURCES:
if value.startswith(settings.BACKEND_PREFIX_ID):
try:
if name == "server":
value = id_from_instance_name(value)
elif name == "network":
value = id_from_network_name(value)
elif name == "port":
value = id_from_nic_name(value)
elif name == "volume":
value = id_from_disk_name(value)
except ValueError:
raise CommandError("Invalid {} ID: {}".format(
capital_name, value))
if for_update:
objects = objects.select_for_update()
try:
return objects.get(id=value)
except ObjectDoesNotExist:
msg = ("{0} with ID {1} does not exist. Use {2}-list to find out"
" available {2} IDs.")
raise CommandError(msg.format(capital_name, value, name))
except (ValueError, TypeError):
raise CommandError("Invalid {} ID: {}".format(capital_name, value))
def wrapper(msg, atomic_context=None):
try:
network_id = utils.id_from_network_name(msg["network"])
network = Network.objects.select_for_update().get(id=network_id)
if network.deleted:
log.debug("Ignoring message for deleted network '%s'", network)
return
backend = Backend.objects.get(clustername=msg['cluster'])
bnet, new = BackendNetwork.objects.get_or_create(network=network,
backend=backend)
if new:
log.info("Created missing BackendNetwork %s", bnet)
func(bnet, msg, atomic_context=atomic_context)
except Network.InvalidBackendIdError:
log.debug("Ignoring msg for unknown network %s.", msg['network'])
except Network.DoesNotExist:
log.error("Network %s not found in DB.", msg['network'])
except Backend.DoesNotExist:
log.error("Backend %s not found in DB.", msg['cluster'])
except BackendNetwork.DoesNotExist:
log.error("Network %s on backend %s not found in DB.",
msg['network'], msg['cluster'])
def process_ganeti_nics(ganeti_nics):
"""Process NIC dict from ganeti"""
new_nics = []
for index, gnic in enumerate(ganeti_nics):
nic_name = gnic.get("name", None)
if nic_name is not None:
nic_id = utils.id_from_nic_name(nic_name)
else:
# Put as default value the index. If it is an unknown NIC to
# synnefo it will be created automaticaly.
nic_id = UNKNOWN_NIC_PREFIX + str(index)
network_name = gnic.get('network', '')
network_id = utils.id_from_network_name(network_name)
network = Network.objects.get(id=network_id)
# Get the new nic info
mac = gnic.get('mac')
ipv4 = gnic.get('ip')
subnet6 = network.subnet6
ipv6 = mac2eui64(mac, subnet6.cidr) if subnet6 else None
firewall = gnic.get('firewall')
firewall_profile = _reverse_tags.get(firewall)
if not firewall_profile and network.public:
firewall_profile = settings.DEFAULT_FIREWALL_PROFILE
nic_info = {
'index': index,
'network': network,
'mac': mac,
'ipv4_address': ipv4,
'ipv6_address': ipv6,
'firewall_profile': firewall_profile,
'state': 'ACTIVE'
}
new_nics.append((nic_id, nic_info))
return dict(new_nics)
def process_ganeti_nics(ganeti_nics):
"""Process NIC dict from ganeti"""
new_nics = []
for index, gnic in enumerate(ganeti_nics):
nic_name = gnic.get("name", None)
if nic_name is not None:
nic_id = utils.id_from_nic_name(nic_name)
else:
# Put as default value the index. If it is an unknown NIC to
# synnefo it will be created automaticaly.
nic_id = UNKNOWN_NIC_PREFIX + str(index)
network_name = gnic.get('network', '')
network_id = utils.id_from_network_name(network_name)
network = Network.objects.get(id=network_id)
# Get the new nic info
mac = gnic.get('mac')
ipv4 = gnic.get('ip')
subnet6 = network.subnet6
ipv6 = mac2eui64(mac, subnet6.cidr) if subnet6 else None
firewall = gnic.get('firewall')
firewall_profile = _reverse_tags.get(firewall)
if not firewall_profile and network.public:
firewall_profile = settings.DEFAULT_FIREWALL_PROFILE
nic_info = {
'index': index,
'network': network,
'mac': mac,
'ipv4_address': ipv4,
'ipv6_address': ipv6,
'firewall_profile': firewall_profile,
'state': 'ACTIVE'}
new_nics.append((nic_id, nic_info))
return dict(new_nics)
def handle(self, **options):
verbosity = int(options['verbosity'])
self._process_args(options)
backend_id = options['backend-id']
backend = get_backend(backend_id) if backend_id else None
G, GNics = reconciliation.get_instances_from_ganeti(backend)
D = reconciliation.get_servers_from_db(backend)
DBNics = reconciliation.get_nics_from_db(backend)
#
# Detect problems
#
if options['detect_stale']:
stale = reconciliation.stale_servers_in_db(D, G)
if len(stale) > 0:
print >> sys.stderr, "Found the following stale server IDs: "
print " " + "\n ".join([str(x) for x in stale])
elif verbosity == 2:
print >> sys.stderr, "Found no stale server IDs in DB."
if options['detect_orphans']:
orphans = reconciliation.orphan_instances_in_ganeti(D, G)
if len(orphans) > 0:
print >> sys.stderr, "Found orphan Ganeti instances with IDs: "
print " " + "\n ".join([str(x) for x in orphans])
elif verbosity == 2:
print >> sys.stderr, "Found no orphan Ganeti instances."
if options['detect_unsynced']:
unsynced = reconciliation.unsynced_operstate(D, G)
if len(unsynced) > 0:
print >> sys.stderr, "The operstate of the following server" \
" IDs is out-of-sync:"
print " " + "\n ".join([
"%d is %s in DB, %s in Ganeti" %
(x[0], x[1], ('UP' if x[2] else 'DOWN')) for x in unsynced
])
elif verbosity == 2:
print >> sys.stderr, "The operstate of all servers is in sync."
if options['detect_build_errors']:
build_errors = reconciliation.instances_with_build_errors(D, G)
if len(build_errors) > 0:
msg = "The os for the following server IDs was not build"\
" successfully:"
print >> sys.stderr, msg
print " " + "\n ".join(["%d" % x for x in build_errors])
elif verbosity == 2:
print >> sys.stderr, "Found no instances with build errors."
if options['detect_unsynced_nics']:
def pretty_print_nics(nics):
if not nics:
print ''.ljust(18) + 'None'
for index, info in nics.items():
print ''.ljust(18) + 'nic/' + str(index) +\
': MAC: %s, IP: %s, Network: %s' % \
(info['mac'], info['ipv4'], info['network'])
unsynced_nics = reconciliation.unsynced_nics(DBNics, GNics)
if len(unsynced_nics) > 0:
msg = "The NICs of the servers with the following IDs are"\
" unsynced:"
print >> sys.stderr, msg
for id, nics in unsynced_nics.items():
print ''.ljust(2) + '%6d:' % id
print ''.ljust(8) + '%8s:' % 'DB'
pretty_print_nics(nics[0])
print ''.ljust(8) + '%8s:' % 'Ganeti'
pretty_print_nics(nics[1])
elif verbosity == 2:
print >> sys.stderr, "All instance nics are synced."
#
# Then fix them
#
if options['fix_stale'] and len(stale) > 0:
print >> sys.stderr, \
"Simulating successful Ganeti removal for %d " \
"servers in the DB:" % len(stale)
for vm in VirtualMachine.objects.filter(pk__in=stale):
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm,
etime=event_time,
jobid=-0,
opcode='OP_INSTANCE_REMOVE',
status='success',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_orphans'] and len(orphans) > 0:
print >> sys.stderr, \
"Issuing OP_INSTANCE_REMOVE for %d Ganeti instances:" % \
len(orphans)
for id in orphans:
try:
vm = VirtualMachine.objects.get(pk=id)
with pooled_rapi_client(vm) as client:
client.DeleteInstance(utils.id_to_instance_name(id))
except VirtualMachine.DoesNotExist:
print >> sys.stderr, "No entry for VM %d in DB !!" % id
print >> sys.stderr, " ...done"
if options['fix_unsynced'] and len(unsynced) > 0:
print >> sys.stderr, "Setting the state of %d out-of-sync VMs:" % \
len(unsynced)
for id, db_state, ganeti_up in unsynced:
vm = VirtualMachine.objects.get(pk=id)
opcode = "OP_INSTANCE_REBOOT" if ganeti_up \
else "OP_INSTANCE_SHUTDOWN"
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm,
etime=event_time,
jobid=-0,
opcode=opcode,
status='success',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_build_errors'] and len(build_errors) > 0:
print >> sys.stderr, "Setting the state of %d build-errors VMs:" %\
len(build_errors)
for id in build_errors:
vm = VirtualMachine.objects.get(pk=id)
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm,
etime=event_time,
jobid=-0,
opcode="OP_INSTANCE_CREATE",
status='error',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_unsynced_nics'] and len(unsynced_nics) > 0:
print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \
len(unsynced_nics)
for id, nics in unsynced_nics.items():
vm = VirtualMachine.objects.get(pk=id)
nics = nics[1] # Ganeti nics
if nics == {}: # No nics
vm.nics.all.delete()
continue
for index, nic in nics.items():
net_id = utils.id_from_network_name(nic['network'])
subnet6 = Network.objects.get(id=net_id).subnet6
# Produce ipv6
ipv6 = subnet6 and mac2eui64(nic['mac'], subnet6) or None
nic['ipv6'] = ipv6
# Rename ipv4 to ip
nic['ip'] = nic['ipv4']
# Dict to sorted list
final_nics = []
nics_keys = nics.keys()
nics_keys.sort()
for i in nics_keys:
if nics[i]['network']:
final_nics.append(nics[i])
else:
print 'Network of nic %d of vm %s is None. ' \
'Can not reconcile' % (i, vm.backend_vm_id)
event_time = datetime.datetime.now()
backend_mod.process_net_status(vm=vm,
etime=event_time,
nics=final_nics)
print >> sys.stderr, " ...done"
def test_id_from_net_name(self):
self.assertEqual(utils.id_from_network_name('snf-net-42'), 42)
for name in [None, 'foo', 'snf42', 'snf-net-a', 'snf-snf-42', 1234]:
self.assertRaises(Network.InvalidBackendIdError,
utils.id_from_network_name, '')
def test_id_from_net_name(self):
self.assertEqual(utils.id_from_network_name('snf-net-42'), 42)
for name in [None, 'foo', 'snf42', 'snf-net-a', 'snf-snf-42', 1234]:
self.assertRaises(Network.InvalidBackendIdError,
utils.id_from_network_name, '')
def handle(self, **options):
verbosity = int(options['verbosity'])
self._process_args(options)
backend_id = options['backend-id']
backend = get_backend(backend_id) if backend_id else None
G, GNics = reconciliation.get_instances_from_ganeti(backend)
D = reconciliation.get_servers_from_db(backend)
DBNics = reconciliation.get_nics_from_db(backend)
#
# Detect problems
#
if options['detect_stale']:
stale = reconciliation.stale_servers_in_db(D, G)
if len(stale) > 0:
print >> sys.stderr, "Found the following stale server IDs: "
print " " + "\n ".join(
[str(x) for x in stale])
elif verbosity == 2:
print >> sys.stderr, "Found no stale server IDs in DB."
if options['detect_orphans']:
orphans = reconciliation.orphan_instances_in_ganeti(D, G)
if len(orphans) > 0:
print >> sys.stderr, "Found orphan Ganeti instances with IDs: "
print " " + "\n ".join(
[str(x) for x in orphans])
elif verbosity == 2:
print >> sys.stderr, "Found no orphan Ganeti instances."
if options['detect_unsynced']:
unsynced = reconciliation.unsynced_operstate(D, G)
if len(unsynced) > 0:
print >> sys.stderr, "The operstate of the following server" \
" IDs is out-of-sync:"
print " " + "\n ".join(
["%d is %s in DB, %s in Ganeti" %
(x[0], x[1], ('UP' if x[2] else 'DOWN'))
for x in unsynced])
elif verbosity == 2:
print >> sys.stderr, "The operstate of all servers is in sync."
if options['detect_build_errors']:
build_errors = reconciliation.instances_with_build_errors(D, G)
if len(build_errors) > 0:
msg = "The os for the following server IDs was not build"\
" successfully:"
print >> sys.stderr, msg
print " " + "\n ".join(
["%d" % x for x in build_errors])
elif verbosity == 2:
print >> sys.stderr, "Found no instances with build errors."
if options['detect_unsynced_nics']:
def pretty_print_nics(nics):
if not nics:
print ''.ljust(18) + 'None'
for index, info in nics.items():
print ''.ljust(18) + 'nic/' + str(index) +\
': MAC: %s, IP: %s, Network: %s' % \
(info['mac'], info['ipv4'], info['network'])
unsynced_nics = reconciliation.unsynced_nics(DBNics, GNics)
if len(unsynced_nics) > 0:
msg = "The NICs of the servers with the following IDs are"\
" unsynced:"
print >> sys.stderr, msg
for id, nics in unsynced_nics.items():
print ''.ljust(2) + '%6d:' % id
print ''.ljust(8) + '%8s:' % 'DB'
pretty_print_nics(nics[0])
print ''.ljust(8) + '%8s:' % 'Ganeti'
pretty_print_nics(nics[1])
elif verbosity == 2:
print >> sys.stderr, "All instance nics are synced."
#
# Then fix them
#
if options['fix_stale'] and len(stale) > 0:
print >> sys.stderr, \
"Simulating successful Ganeti removal for %d " \
"servers in the DB:" % len(stale)
for vm in VirtualMachine.objects.filter(pk__in=stale):
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm,
etime=event_time,
jobid=-0,
opcode='OP_INSTANCE_REMOVE', status='success',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_orphans'] and len(orphans) > 0:
print >> sys.stderr, \
"Issuing OP_INSTANCE_REMOVE for %d Ganeti instances:" % \
len(orphans)
for id in orphans:
try:
vm = VirtualMachine.objects.get(pk=id)
with pooled_rapi_client(vm) as client:
client.DeleteInstance(utils.id_to_instance_name(id))
except VirtualMachine.DoesNotExist:
print >> sys.stderr, "No entry for VM %d in DB !!" % id
print >> sys.stderr, " ...done"
if options['fix_unsynced'] and len(unsynced) > 0:
print >> sys.stderr, "Setting the state of %d out-of-sync VMs:" % \
len(unsynced)
for id, db_state, ganeti_up in unsynced:
vm = VirtualMachine.objects.get(pk=id)
opcode = "OP_INSTANCE_REBOOT" if ganeti_up \
else "OP_INSTANCE_SHUTDOWN"
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm, etime=event_time, jobid=-0,
opcode=opcode, status='success',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_build_errors'] and len(build_errors) > 0:
print >> sys.stderr, "Setting the state of %d build-errors VMs:" %\
len(build_errors)
for id in build_errors:
vm = VirtualMachine.objects.get(pk=id)
event_time = datetime.datetime.now()
backend_mod.process_op_status(
vm=vm, etime=event_time, jobid=-0,
opcode="OP_INSTANCE_CREATE", status='error',
logmsg='Reconciliation: simulated Ganeti event')
print >> sys.stderr, " ...done"
if options['fix_unsynced_nics'] and len(unsynced_nics) > 0:
print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \
len(unsynced_nics)
for id, nics in unsynced_nics.items():
vm = VirtualMachine.objects.get(pk=id)
nics = nics[1] # Ganeti nics
if nics == {}: # No nics
vm.nics.all.delete()
continue
for index, nic in nics.items():
net_id = utils.id_from_network_name(nic['network'])
subnet6 = Network.objects.get(id=net_id).subnet6
# Produce ipv6
ipv6 = subnet6 and mac2eui64(nic['mac'], subnet6) or None
nic['ipv6'] = ipv6
# Rename ipv4 to ip
nic['ip'] = nic['ipv4']
# Dict to sorted list
final_nics = []
nics_keys = nics.keys()
nics_keys.sort()
for i in nics_keys:
if nics[i]['network']:
final_nics.append(nics[i])
else:
print 'Network of nic %d of vm %s is None. ' \
'Can not reconcile' % (i, vm.backend_vm_id)
event_time = datetime.datetime.now()
backend_mod.process_net_status(vm=vm, etime=event_time,
nics=final_nics)
print >> sys.stderr, " ...done"
