def main(argv=None):
if argv is None:
argv = sys.argv
parser = base_parser('Remove orphan ports in Neutron referring to an '
'inactive Ironic instance')
parser.add_argument('mode', choices=['info', 'delete'],
help='Just display data on the conflict ports or delete them')
parser.add_argument('--ignore-subnet', type=str,
help='Ignore Neutron ports in this subnet (UUID). Must provide either '
'this or --ignore-from-ironic-conf. This overrides the conf.')
parser.add_argument('-c', '--ignore-from-ironic-conf', type=str,
help='Ignore Neutron ports in the subnet(s) under the '
'"provisioning_network" network in the "neutron" section of '
'this configuration file.')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--force-sane', action='store_true',
help='Disable sanity checking (i.e. things really are that bad)')
args = parser.parse_args(argv[1:])
# Validate args
slack = Slackbot(args.slack, SUBCOMMAND) if args.slack else None
auth = Auth.from_env_or_args(args=args)
if args.ignore_subnet:
ignore_subnets = [args.ignore_subnet]
elif args.ignore_from_ironic_conf:
ironic_config = configparser.ConfigParser()
ironic_config.read(args.ignore_from_ironic_conf)
net_id = ironic_config['neutron']['provisioning_network']
network = osrest.neutron.network(auth, net_id)
ignore_subnets = network['subnets']
else:
print('Must provide --ignore-subnet or --ignore-from-ironic-conf',
file=sys.stderr)
return -1
# Do actual work
try:
conflict_macs = find_conflicts(auth, ignore_subnets)
if args.mode == 'info':
show_info(conflict_macs)
elif args.mode == 'delete':
if (not args.force_sane) and len(conflict_macs) > 10:
raise RuntimeError('(in)sanity check: thinks there are {} conflicting MACs'.format(len(conflict_macs)))
for mac in conflict_macs.values():
osrest.neutron_port_delete(auth, mac['neutron_port_id'])
if slack:
message = 'Fixed Ironic/Neutron MAC conflicts\n{}'.format(
'\n'.join(
' • Neutron Port `{neutron_port_id}` → `{mac}` ← Ironic Node `{ironic_node_id}` (Port `{ironic_port}`)'
.format(**m) for m in conflict_macs.values()
)
)
slack.success(message)
else:
print('unknown command', file=sys.stderr)
return -1
except:
if slack:
slack.exception()
raise
def main(argv=None):
if argv is None:
argv = sys.argv
parser = base_parser(
'Kick Ironic nodes that refer to a deleted/nonexistant Nova instance')
parser.add_argument(
'mode',
choices=['info', 'delete'],
help='Just display data on the bound nodes or delete them')
parser.add_argument(
'--slack',
type=str,
help=
'JSON file with Slack webhook information to send a notification to')
parser.add_argument(
'--osrc',
type=str,
help='Connection parameter file. Should include password. envars used '
'if not provided by this file.')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument(
'--force-sane',
action='store_true',
help='Disable sanity checking (i.e. things really are that bad)')
parser.add_argument('--force-insane',
action='store_true',
help=argparse.SUPPRESS) # for testing
args = parser.parse_args(argv[1:])
slack = Slackbot(args.slack,
script_name='undead-instances') if args.slack else None
os_vars = {
k: os.environ[k]
for k in os.environ if k.startswith(OS_ENV_PREFIX)
}
if args.osrc:
os_vars.update(load_osrc(args.osrc))
missing_os_vars = set(Auth.required_os_vars) - set(os_vars)
if missing_os_vars:
print('Missing required OS values in env/rcfile: {}'.format(
', '.join(missing_os_vars)),
file=sys.stderr)
return -1
auth = Auth(os_vars)
nodes = osrest.ironic_nodes(auth)
instances = osrest.nova_instances(auth)
node_instance_map, unbound_instances = find_unbound_instances(
auth, nodes, instances)
if args.mode == 'info':
# no-op
if unbound_instances:
print('ZOMBIE INSTANCES ON NODES')
else:
print('No zombies currently.')
for inst_id in unbound_instances:
node = node_instance_map[inst_id]
assert inst_id not in instances, 'contradiction, this should be impossible'
print('-----')
print('Ironic Node\n' ' ID: {}'.format(node['uuid']))
print(' Instance: {}'.format(node['instance_uuid']))
print(' State: {}'.format(node['provision_state']))
elif args.mode == 'delete':
if not args.force_sane or args.force_insane:
# sanity check(s) to avoid doing something stupid
if len(instance_ids) == 0 and len(unbound_instances) != 0:
_thats_crazy('(in)sanity check: 0 running instances(?!)',
slack)
ubi_limit = 20 if not args.force_insane else -1
if len(unbound_instances) > ubi_limit:
_thats_crazy(
'(in)sanity check: it thinks there are {} unbound instances'
.format(len(unbound_instances)),
slack,
)
try:
for inst_id in unbound_instances:
node = node_instance_map[inst_id]
node_id = node['uuid']
if node['provision_state'] == 'available':
clear_node_instance_data(auth, node_id)
else:
osrest.ironic_node_set_state(auth, node_id, 'deleted')
message = 'Fixed Ironic nodes with nonexistant instances:\n{}'.format(
'\n'.join(' • node `{}` → instance `{}`'.format(
node_instance_map[i]['uuid'], node_instance_map[i]
['instance_uuid']) for i in unbound_instances))
print(message)
if slack:
slack.success(message)
except:
if slack:
slack.exception()
raise
def main(argv=None):
if argv is None:
argv = sys.argv
parser = base_parser(
'Kick Ironic nodes that are in an common/known error state')
parser.add_argument(
'mode',
choices=['info', 'reset'],
help='Just display data on the stuck nodes or reset their states')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--dry-run',
action='store_true',
help='Dry run, don\'t actually do anything')
args = parser.parse_args(argv[1:])
slack = Slackbot(
args.slack,
script_name='ironic-error-resetter') if args.slack else None
os_vars = {
k: os.environ[k]
for k in os.environ if k.startswith(OS_ENV_PREFIX)
}
if args.osrc:
os_vars.update(load_osrc(args.osrc))
missing_os_vars = set(Auth.required_os_vars) - set(os_vars)
if missing_os_vars:
print('Missing required OS values in env/rcfile: {}'.format(
', '.join(missing_os_vars)),
file=sys.stderr)
return -1
auth = Auth(os_vars)
try:
nodes = osrest.ironic_nodes(auth, details=True)
cureable = cureable_nodes(nodes)
if args.mode == 'info':
print('{} node(s) in a state that we can treat'.format(
len(cureable)))
for nid in cureable:
print('-' * 40)
print('\n'.join('{:<25s} {}'.format(key, nodes[nid].get(key))
for key in [
'uuid',
'provision_updated_at',
'provision_state',
'last_error',
'instance_uuid',
'extra',
'maintenance',
]))
return
if len(cureable) == 0:
if args.verbose:
print('Nothing to do.')
return
print('To correct: {}'.format(repr(cureable)))
reset_ok = []
too_many = []
for nid in cureable:
resetter = NodeResetter(auth, nid, dry_run=args.dry_run)
resetter.reset()
reset_ok.append((nid, resetter.tracker.count()))
message_lines = []
if reset_ok:
message_lines.append('Performed reset of nodes')
message_lines.extend(' • `{}`: {} resets'.format(*r)
for r in reset_ok)
if too_many:
message_lines.append('Skipped (already at limit)')
message_lines.extend(' • `{}`'.format(r) for r in too_many)
if args.dry_run:
message_lines.append('dry run, no changes actually made.')
message = '\n'.join(message_lines)
print(message)
if slack and (not args.dry_run):
slack.success(message)
except:
if slack:
slack.exception()
raise
def main(argv=None):
if argv is None:
argv = sys.argv
parser = base_parser('Floating IP and port reclaimer.')
mysqlargs = MySqlArgs({
'user': '******',
'password': '',
'host': 'localhost',
'port': 3306,
})
mysqlargs.inject(parser)
parser.add_argument(
'-q',
'--quiet',
action='store_true',
help='Quiet mode. No output to Slack if there was nothing to do.')
parser.add_argument(
'--multiport',
action='store_true',
help='Enable if Ironic nodes may have multiple ports associated.')
parser.add_argument('action',
choices=['info', 'clean'],
help='Just display info or actually fix them?')
args = parser.parse_args(argv[1:])
mysqlargs.extract(args)
auth = osapi.Auth.from_env_or_args(args=args)
slack = Slackbot(args.slack,
script_name='dirty-ports') if args.slack else None
assert_single = False if args.multiport else True
take_action = args.action == 'clean'
db = mysqlargs.connect()
try:
bad_ports = identify_dirty_ports(auth, assert_single)
if bad_ports:
str_ports = '\n'.join(
' • port `{uuid}` on node `{node_uuid}`'.format(**p)
for p in bad_ports)
if take_action:
clean_ports(db, bad_ports)
message = "Cleaned {} ports with `internal_info` data on `available` nodes:\n{}".format(
len(bad_ports), str_ports)
print(message)
if slack:
slack.success(message)
else:
print("(read-only mode, not cleaning ports):\n{}".format(
str_ports))
except:
if slack:
slack.exception()
raise