def handle(self, *args, **options):
write = self.stderr.write
fix = options["fix"]
force = options["force"]
handlers = self.get_handlers(options["resources"])
maxops = options["max_operations"]
if maxops is not None:
try:
maxops = int(maxops)
except ValueError:
m = "Expected integer max operations."
raise CommandError(m)
shutdown_timeout = options["shutdown_timeout"]
if shutdown_timeout is not None:
try:
shutdown_timeout = int(shutdown_timeout)
except ValueError:
m = "Expected integer shutdown timeout."
raise CommandError(m)
remove_system_volumes = options["remove_system_volumes"]
cascade_remove = options["cascade_remove"]
excluded_users = options['exclude_users']
excluded_users = set(excluded_users.split(',')
if excluded_users is not None else [])
users_to_check = options['users']
if users_to_check is not None:
users_to_check = list(set(users_to_check.split(',')) -
excluded_users)
try:
qh_holdings = util.get_qh_users_holdings(users_to_check)
except errors.AstakosClientException as e:
raise CommandError(e)
excluded_projects = options["exclude_projects"]
excluded_projects = set(excluded_projects.split(',')
if excluded_projects is not None else [])
projects_to_check = options["projects"]
if projects_to_check is not None:
projects_to_check = list(set(projects_to_check.split(',')) -
excluded_projects)
try:
qh_project_holdings = util.get_qh_project_holdings(
projects_to_check)
except errors.AstakosClientException as e:
raise CommandError(e)
qh_project_holdings = sorted(qh_project_holdings.items())
qh_holdings = sorted(qh_holdings.items())
resources = set(h[0] for h in handlers)
dangerous = bool(resources.difference(DEFAULT_RESOURCES))
self.stderr.write("Checking resources %s...\n" %
",".join(list(resources)))
hopts = {"cascade_remove": cascade_remove,
"remove_system_volumes": remove_system_volumes,
}
opts = {"shutdown_timeout": shutdown_timeout}
actions = {}
overlimit = []
viol_id = 0
remains = defaultdict(list)
if users_to_check is None:
for resource, handle_resource, resource_type in handlers:
if resource_type not in actions:
actions[resource_type] = OrderedDict()
actual_resources = enforce.get_actual_resources(
resource_type, projects=projects_to_check)
for project, project_quota in qh_project_holdings:
if enforce.skip_check(project, projects_to_check,
excluded_projects):
continue
try:
qh = util.transform_project_quotas(project_quota)
qh_value, qh_limit, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for project '%s'!\n" %
(resource, project))
continue
if qh_pending:
write("Pending commission for project '%s', "
"resource '%s'. Skipping\n" %
(project, resource))
continue
diff = qh_value - qh_limit
if diff > 0:
viol_id += 1
overlimit.append((viol_id, "project", project, "",
resource, qh_limit, qh_value))
relevant_resources = enforce.pick_project_resources(
actual_resources[project], users=users_to_check,
excluded_users=excluded_users)
handle_resource(viol_id, resource, relevant_resources,
diff, actions, remains, options=hopts)
for resource, handle_resource, resource_type in handlers:
if resource_type not in actions:
actions[resource_type] = OrderedDict()
actual_resources = enforce.get_actual_resources(resource_type,
users_to_check)
for user, user_quota in qh_holdings:
if enforce.skip_check(user, users_to_check, excluded_users):
continue
for source, source_quota in user_quota.iteritems():
if enforce.skip_check(source, projects_to_check,
excluded_projects):
continue
try:
qh = util.transform_quotas(source_quota)
qh_value, qh_limit, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s' and source '%s'!\n" %
(resource, user, source))
continue
if qh_pending:
write("Pending commission for user '%s', source '%s', "
"resource '%s'. Skipping\n" %
(user, source, resource))
continue
diff = qh_value - qh_limit
if diff > 0:
viol_id += 1
overlimit.append((viol_id, "user", user, source,
resource, qh_limit, qh_value))
relevant_resources = actual_resources[source][user]
handle_resource(viol_id, resource, relevant_resources,
diff, actions, remains, options=hopts)
if not overlimit:
write("No violations.\n")
return
headers = ("#", "Type", "Holder", "Source", "Resource", "Limit",
"Usage")
pprint_table(self.stdout, overlimit, headers,
options["output_format"], title="Violations")
if any(actions.values()):
self.stdout.write("\n")
if fix:
if dangerous and not force:
write("You are enforcing resources that may permanently "
"remove a vm or volume.\n")
self.confirm()
write("Applying actions. Please wait...\n")
title = "Applied Actions" if fix else "Suggested Actions"
log = enforce.perform_actions(actions, maxops=maxops, fix=fix,
options=opts)
headers = ("Type", "ID", "State", "Backend", "Action", "Violation")
if fix:
headers += ("Result",)
pprint_table(self.stdout, log, headers,
options["output_format"], title=title)
def explain(resource):
if resource == "cyclades.disk":
if not remove_system_volumes:
return (", because this would need to remove system "
"volumes; if you want to do so, use the "
"--remove-system-volumes option:")
if not cascade_remove:
return (", because this would trigger the removal of "
"attached volumes, too; if you want to do "
"so, use the --cascade-remove option:")
elif resource in DESTROY_RESOURCES:
if not cascade_remove:
return (", because this would trigger the removal of "
"attached volumes, too; if you want to do "
"so, use the --cascade-remove option:")
return ":"
if remains:
self.stderr.write("\n")
for resource, viols in remains.iteritems():
self.stderr.write(
"The following violations for resource '%s' "
"could not be resolved%s\n"
% (resource, explain(resource)))
self.stderr.write(" %s\n" % ",".join(map(str, viols)))
def handle(self, *args, **options):
write = self.stderr.write
fix = options["fix"]
force = options["force"]
maxops = options["max_operations"]
if maxops is not None:
try:
maxops = int(maxops)
except ValueError:
m = "Expected integer max operations."
raise CommandError(m)
shutdown_timeout = options["shutdown_timeout"]
if shutdown_timeout is not None:
try:
shutdown_timeout = int(shutdown_timeout)
except ValueError:
m = "Expected integer shutdown timeout."
raise CommandError(m)
users = options['users']
if users is not None:
users = users.split(',')
excluded = options['exclude_users']
excluded = set(excluded.split(',') if excluded is not None else [])
handlers = self.get_handlers(options["resources"])
try:
qh_holdings = util.get_qh_users_holdings(users)
except errors.AstakosClientException as e:
raise CommandError(e)
qh_holdings = sorted(qh_holdings.items())
resources = set(h[0] for h in handlers)
dangerous = bool(resources.difference(DEFAULT_RESOURCES))
opts = {"shutdown_timeout": shutdown_timeout}
actions = {}
overlimit = []
viol_id = 0
for resource, handle_resource, resource_type in handlers:
if resource_type not in actions:
actions[resource_type] = OrderedDict()
actual_resources = enforce.get_actual_resources(resource_type,
users)
for user, user_quota in qh_holdings:
if user in excluded:
continue
for source, source_quota in user_quota.iteritems():
try:
qh = util.transform_quotas(source_quota)
qh_value, qh_limit, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s' and source '%s'!\n" %
(resource, user, source))
continue
if qh_pending:
write("Pending commission for user '%s', source '%s', "
"resource '%s'. Skipping\n" %
(user, source, resource))
continue
diff = qh_value - qh_limit
if diff > 0:
viol_id += 1
overlimit.append((viol_id, user, source, resource,
qh_limit, qh_value))
relevant_resources = actual_resources[user]
handle_resource(viol_id, resource, relevant_resources,
diff, actions)
if not overlimit:
write("No violations.\n")
return
headers = ("#", "User", "Source", "Resource", "Limit", "Usage")
pprint_table(self.stdout, overlimit, headers,
options["output_format"], title="Violations")
if any(actions.values()):
self.stdout.write("\n")
if fix:
if dangerous and not force:
write("You are enforcing resources that may permanently "
"remove a vm.\n")
self.confirm()
write("Applying actions. Please wait...\n")
title = "Applied Actions" if fix else "Suggested Actions"
log = enforce.perform_actions(actions, maxops=maxops, fix=fix,
options=opts)
headers = ("Type", "ID", "State", "Backend", "Action", "Violation")
if fix:
headers += ("Result",)
pprint_table(self.stdout, log, headers,
options["output_format"], title=title)
def handle(self, *args, **options):
write = self.stdout.write
userid = options['userid']
# Get holdings from Cyclades DB
db_holdings = get_db_holdings(userid)
# Get holdings from QuotaHolder
qh_holdings = get_quotaholder_holdings(userid)
users = set(db_holdings.keys())
users.update(qh_holdings.keys())
# Remove 'None' user
users.discard(None)
if userid and userid not in users:
write("User '%s' does not exist in Quotaholder!", userid)
return
pending_exists = False
unknown_user_exists = False
unsynced = []
for user in users:
db = db_holdings.get(user, {})
try:
qh_all = qh_holdings[user]
except KeyError:
write("User '%s' does not exist in Quotaholder!\n" %
user)
unknown_user_exists = True
continue
# Assuming only one source
qh = qh_all.get(quotas.DEFAULT_SOURCE, {})
qh = transform_quotas(qh)
for resource in quotas.RESOURCES:
db_value = db.pop(resource, 0)
try:
qh_value, _, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s'!\n" % (resource, user))
continue
if qh_pending:
write("Pending commission. User '%s', resource '%s'.\n" %
(user, resource))
pending_exists = True
continue
if db_value != qh_value:
data = (user, resource, db_value, qh_value)
unsynced.append(data)
headers = ("User", "Resource", "Database", "Quotaholder")
if unsynced:
pprint_table(self.stderr, unsynced, headers)
if options["fix"]:
qh = quotas.Quotaholder.get()
request = {}
request["force"] = options["force"]
request["auto_accept"] = True
request["name"] = "RECONCILE"
request["provisions"] = map(create_provision, unsynced)
try:
qh.issue_commission(ASTAKOS_TOKEN, request)
except quotas.QuotaLimit:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
return
write("Fixed unsynced resources\n")
if pending_exists:
write("Found pending commissions. Run 'snf-manage"
" reconcile-commissions-cyclades'\n")
elif not (unsynced or unknown_user_exists):
write("Everything in sync.\n")
def handle(self, *args, **options):
write = self.stderr.write
if options["list_resources"]:
self.help_resources(options)
exit()
fix = options["fix"]
force = options["force"]
handlers = self.get_handlers(options["resources"],
options["soft_resources"])
maxops = options["max_operations"]
if maxops is not None:
try:
maxops = int(maxops)
except ValueError:
m = "Expected integer max operations."
raise CommandError(m)
shutdown_timeout = options["shutdown_timeout"]
if shutdown_timeout is not None:
try:
shutdown_timeout = int(shutdown_timeout)
except ValueError:
m = "Expected integer shutdown timeout."
raise CommandError(m)
remove_system_volumes = options["remove_system_volumes"]
cascade_remove = options["cascade_remove"]
excluded_users = options['exclude_users']
excluded_users = set(
excluded_users.split(',') if excluded_users is not None else [])
users_to_check = options['users']
if users_to_check is not None:
users_to_check = list(
set(users_to_check.split(',')) - excluded_users)
try:
qh_holdings = util.get_qh_users_holdings(users_to_check)
except errors.AstakosClientException as e:
raise CommandError(e)
excluded_projects = options["exclude_projects"]
excluded_projects = set(
excluded_projects.split(','
) if excluded_projects is not None else [])
projects_to_check = options["projects"]
if projects_to_check is not None:
projects_to_check = list(
set(projects_to_check.split(',')) - excluded_projects)
try:
qh_project_holdings = util.get_qh_project_holdings(
projects_to_check)
except errors.AstakosClientException as e:
raise CommandError(e)
qh_project_holdings = sorted(qh_project_holdings.items())
qh_holdings = sorted(qh_holdings.items())
resources = set(h[0] for h in handlers if not h[1])
dangerous = bool(resources.intersection(DESTROY_RESOURCES))
if resources:
self.stderr.write("Checking resources %s...\n" %
",".join(list(resources)))
resources_soft = set(h[0] for h in handlers if h[1])
if resources_soft:
self.stderr.write("Checking resources for soft enforce %s...\n" %
",".join(list(resources_soft)))
if not resources and not resources_soft:
self.stderr.write("No resources specified; use '--list-resources' "
"to list available resources.\n")
exit()
hopts = {
"cascade_remove": cascade_remove,
"remove_system_volumes": remove_system_volumes,
}
opts = {"shutdown_timeout": shutdown_timeout}
actions = {}
overlimit = []
viol_id = 0
remains = defaultdict(list)
if users_to_check is None:
for resource, is_soft, handle_resource, resource_type in handlers:
if resource_type not in actions:
actions[resource_type] = OrderedDict()
actual_resources = enforce.get_actual_resources(
resource_type, projects=projects_to_check)
for project, project_quota in qh_project_holdings:
if enforce.skip_check(project, projects_to_check,
excluded_projects):
continue
try:
qh = util.transform_project_quotas(project_quota)
qh_value, qh_limit, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for project '%s'!\n" % (resource, project))
continue
if qh_pending:
write("Pending commission for project '%s', "
"resource '%s'. Skipping\n" %
(project, resource))
continue
diff = qh_value - qh_limit
if diff > 0:
viol_id += 1
overlimit.append((viol_id, "project", project, "",
resource, qh_limit, qh_value))
relevant_resources = enforce.pick_project_resources(
actual_resources[project],
users=users_to_check,
excluded_users=excluded_users)
handle_resource(viol_id,
resource,
relevant_resources,
diff,
actions,
remains,
options=hopts)
for resource, is_soft, handle_resource, resource_type in handlers:
if resource_type not in actions:
actions[resource_type] = OrderedDict()
actual_resources = enforce.get_actual_resources(
resource_type, users_to_check)
for user, user_quota in qh_holdings:
if enforce.skip_check(user, users_to_check, excluded_users):
continue
for source, source_quota in user_quota.iteritems():
if enforce.skip_check(source, projects_to_check,
excluded_projects):
continue
try:
qh = util.transform_quotas(source_quota)
qh_value, qh_limit, qh_pending = qh[resource]
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s' and source '%s'!\n" %
(resource, user, source))
continue
if qh_pending:
write("Pending commission for user '%s', source '%s', "
"resource '%s'. Skipping\n" %
(user, source, resource))
continue
diff = qh_value - qh_limit
if diff > 0:
viol_id += 1
overlimit.append((viol_id, "user", user, source,
resource, qh_limit, qh_value))
relevant_resources = actual_resources[source][user]
handle_resource(viol_id,
resource,
relevant_resources,
diff,
actions,
remains,
options=hopts)
if not overlimit:
write("No violations.\n")
return
headers = ("#", "Type", "Holder", "Source", "Resource", "Limit",
"Usage")
pprint_table(self.stdout,
overlimit,
headers,
options["output_format"],
title="Violations")
if any(actions.values()):
self.stdout.write("\n")
if fix:
if dangerous and not force:
write("You are enforcing resources that may permanently "
"remove a VM, volume, or IP.\n")
self.confirm()
write("Applying actions. Please wait...\n")
title = "Applied Actions" if fix else "Suggested Actions"
log = enforce.perform_actions(actions,
maxops=maxops,
fix=fix,
options=opts)
headers = ("Type", "ID", "State", "Backend", "Action", "Violation")
if fix:
headers += ("Result", )
pprint_table(self.stdout,
log,
headers,
options["output_format"],
title=title)
def explain(resource):
if resource == "cyclades.disk":
if not remove_system_volumes:
return (", because this would need to remove system "
"volumes; if you want to do so, use the "
"--remove-system-volumes option:")
if not cascade_remove:
return (", because this would trigger the removal of "
"attached volumes, too; if you want to do "
"so, use the --cascade-remove option:")
elif resource in DESTROY_RESOURCES:
if not cascade_remove:
return (", because this would trigger the removal of "
"attached volumes, too; if you want to do "
"so, use the --cascade-remove option:")
return ":"
if remains:
self.stderr.write("\n")
for resource, viols in remains.iteritems():
self.stderr.write("The following violations for resource '%s' "
"could not be resolved%s\n" %
(resource, explain(resource)))
self.stderr.write(" %s\n" % ",".join(map(str, viols)))
