def _issue_commission(clientkey, provisions, name, force, accept):
serial = qh.issue_commission(clientkey=clientkey,
provisions=provisions,
name=name,
force=force)
if accept:
qh.resolve_pending_commission(clientkey=clientkey, serial=serial)
return serial
def _issue_commission(clientkey, provisions, name, force, accept):
serial = qh.issue_commission(clientkey=clientkey,
provisions=provisions,
name=name,
force=force)
if accept:
qh.resolve_pending_commission(clientkey=clientkey, serial=serial)
return serial
def register_pending_apps(user, quantity, force=False):
provision = (user.uuid, SYSTEM, PENDING_APP_RESOURCE), quantity
try:
s = qh.issue_commission(clientkey='astakos',
force=force,
provisions=[provision])
except NoCapacityError as e:
limit = e.data['limit']
return False, limit
qh.resolve_pending_commission('astakos', s)
return True, None
def register_pending_apps(user, quantity, force=False):
provision = (user.uuid, SYSTEM, PENDING_APP_RESOURCE), quantity
try:
s = qh.issue_commission(clientkey='astakos',
force=force,
provisions=[provision])
except NoCapacityError as e:
limit = e.data['limit']
return False, limit
qh.resolve_pending_commission('astakos', s)
return True, None
def register_pending_apps(triples, force=False):
values = [(get_user_ref(user), get_project_ref(project),
PENDING_APP_RESOURCE, quantity)
for (user, project, quantity) in triples]
provisions = _mk_provisions(values)
try:
s = qh.issue_commission(clientkey='astakos',
force=force,
provisions=provisions)
except NoCapacityError as e:
limit = e.data['limit']
return False, limit
qh.resolve_pending_commission('astakos', s)
return True, None
def register_pending_apps(triples, force=False):
values = [(get_user_ref(user), get_project_ref(project),
PENDING_APP_RESOURCE, quantity)
for (user, project, quantity) in triples]
provisions = _mk_provisions(values)
try:
s = qh.issue_commission(clientkey='astakos',
force=force,
provisions=provisions)
except NoCapacityError as e:
limit = e.data['limit']
return False, limit
qh.resolve_pending_commission('astakos', s)
return True, None
def serial_action(request, serial):
input_data = utils.get_json_body(request)
check_is_dict(input_data)
try:
serial = int(serial)
except ValueError:
raise BadRequest("Serial should be an integer.")
client_key = unicode(request.component_instance)
accept = 'accept' in input_data
reject = 'reject' in input_data
if accept == reject:
raise BadRequest('Specify either accept or reject action.')
result = qh.resolve_pending_commission(clientkey=client_key,
serial=serial,
accept=accept)
response = HttpResponse()
if not result:
response.status_code = 404
return response
def serial_action(request, serial):
data = request.body
try:
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
try:
serial = int(serial)
except ValueError:
raise BadRequest("Serial should be an integer.")
client_key = str(request.component_instance)
accept = 'accept' in input_data
reject = 'reject' in input_data
if accept == reject:
raise BadRequest('Specify either accept or reject action.')
result = qh.resolve_pending_commission(clientkey=client_key,
serial=serial,
accept=accept)
response = HttpResponse()
if not result:
response.status_code = 404
return response
def serial_action(request, serial):
data = request.body
try:
input_data = json.loads(data)
except json.JSONDecodeError:
raise BadRequest("POST data should be in json format.")
try:
serial = int(serial)
except ValueError:
raise BadRequest("Serial should be an integer.")
client_key = str(request.component_instance)
accept = 'accept' in input_data
reject = 'reject' in input_data
if accept == reject:
raise BadRequest('Specify either accept or reject action.')
result = qh.resolve_pending_commission(clientkey=client_key,
serial=serial,
accept=accept)
response = HttpResponse()
if not result:
response.status_code = 404
return response
def serial_action(request, serial):
input_data = utils.get_json_body(request)
check_is_dict(input_data)
try:
serial = int(serial)
except ValueError:
raise BadRequest("Serial should be an integer.")
client_key = unicode(request.component_instance)
accept = 'accept' in input_data
reject = 'reject' in input_data
if accept == reject:
raise BadRequest('Specify either accept or reject action.')
result = qh.resolve_pending_commission(clientkey=client_key,
serial=serial,
accept=accept)
response = HttpResponse()
if not result:
response.status_code = 404
return response
def handle(self, *args, **options):
write = self.stdout.write
force = options['force']
userid = options['userid']
try:
astakos = Component.objects.get(name="astakos")
except Component.DoesNotExist:
raise CommandError("Component 'astakos' not found.")
query = [userid] if userid is not None else None
qh_holdings = service_get_quotas(astakos, query)
if userid is None:
users = AstakosUser.objects.accepted()
else:
try:
user = AstakosUser.objects.get(uuid=userid)
except AstakosUser.DoesNotExist:
raise CommandError("There is no user with uuid '%s'." % userid)
if not user.is_accepted():
raise CommandError("%s is not an accepted user." % userid)
users = [user]
db_holdings = count_pending_app(users)
pending_exists = False
unknown_user_exists = False
unsynced = []
for user in users:
uuid = user.uuid
db_value = db_holdings.get(uuid, 0)
try:
qh_all = qh_holdings[uuid]
except KeyError:
write("User '%s' does not exist in Quotaholder!\n" % uuid)
unknown_user_exists = True
continue
# Assuming only one source
system_qh = qh_all.get(SYSTEM, {})
# Assuming only one resource
resource = 'astakos.pending_app'
try:
qh_values = system_qh[resource]
qh_value = qh_values['usage']
qh_pending = qh_values['pending']
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s'!\n" % (resource, uuid))
continue
if qh_pending:
write("Pending commission. User '%s', resource '%s'.\n" %
(uuid, resource))
pending_exists = True
continue
if db_value != qh_value:
data = (uuid, resource, db_value, qh_value)
unsynced.append(data)
headers = ("User", "Resource", "Astakos", "Quotaholder")
if unsynced:
pprint_table(self.stderr, unsynced, headers)
if options["fix"]:
provisions = map(create_provision, unsynced)
try:
s = qh.issue_commission('astakos', provisions,
name='RECONCILE', force=force)
except qh_exception.NoCapacityError:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
return
qh.resolve_pending_commission('astakos', s)
write("Fixed unsynced resources\n")
if pending_exists:
write("Found pending commissions. "
"This is probably a bug. Please report.\n")
elif not (unsynced or unknown_user_exists):
write("Everything in sync.\n")
def handle(self, *args, **options):
write = self.stdout.write
force = options['force']
userid = options['userid']
try:
astakos = Component.objects.get(name="astakos")
except Component.DoesNotExist:
raise CommandError("Component 'astakos' not found.")
query = [userid] if userid is not None else None
qh_holdings = service_get_quotas(astakos, query)
if userid is None:
users = AstakosUser.objects.verified()
else:
try:
users = [AstakosUser.objects.get(uuid=userid)]
except AstakosUser.DoesNotExist:
raise CommandError("There is no user with uuid '%s'." % userid)
db_holdings = count_pending_app(users)
pending_exists = False
unknown_user_exists = False
unsynced = []
for user in users:
uuid = user.uuid
db_value = db_holdings.get(uuid, 0)
try:
qh_all = qh_holdings[uuid]
except KeyError:
write("User '%s' does not exist in Quotaholder!\n" % uuid)
unknown_user_exists = True
continue
# Assuming only one source
system_qh = qh_all.get(SYSTEM, {})
# Assuming only one resource
resource = 'astakos.pending_app'
try:
qh_values = system_qh[resource]
qh_value = qh_values['usage']
qh_pending = qh_values['pending']
except KeyError:
write("Resource '%s' does not exist in Quotaholder"
" for user '%s'!\n" % (resource, uuid))
continue
if qh_pending:
write("Pending commission. User '%s', resource '%s'.\n" %
(uuid, resource))
pending_exists = True
continue
if db_value != qh_value:
data = (uuid, resource, db_value, qh_value)
unsynced.append(data)
headers = ("User", "Resource", "Astakos", "Quotaholder")
if unsynced:
pprint_table(self.stderr, unsynced, headers)
if options["fix"]:
provisions = map(create_provision, unsynced)
try:
s = qh.issue_commission('astakos',
provisions,
name='RECONCILE',
force=force)
except qh_exception.NoCapacityError:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
return
qh.resolve_pending_commission('astakos', s)
write("Fixed unsynced resources\n")
if pending_exists:
write("Found pending commissions. "
"This is probably a bug. Please report.\n")
elif not (unsynced or unknown_user_exists):
write("Everything in sync.\n")
def test_010_qh(self):
holder = 'h0'
source = 'system'
resource1 = 'r1'
resource2 = 'r2'
limit1 = 10
limit2 = 20
qh.set_quota([((holder, source, resource1), limit1),
((holder, source, resource1), limit2)])
r = qh.get_quota(holders=[holder], sources=[source],
resources=[resource1, resource2])
self.assertEqual(r, {(holder, source, resource1): (limit2, 0, 0)})
r = qh.get_quota()
self.assertEqual(r, {(holder, source, resource1): (limit2, 0, 0)})
# issuing commissions
qh.set_quota([((holder, source, resource1), limit1),
((holder, source, resource2), limit2)])
s1 = self.issue_commission([((holder, source, resource1), limit1/2),
((holder, source, resource2), limit2)],
name="initial")
assertGreater(s1, 0)
r = qh.get_commission(self.client, s1)
provisions = [
{'holder': holder,
'source': source,
'resource': resource1,
'quantity': limit1/2,
},
{'holder': holder,
'source': source,
'resource': resource2,
'quantity': limit2,
}
]
self.assertEqual(r['serial'], s1)
ps = r['provisions']
for p in ps:
assertIn(p, provisions)
with assertRaises(NoCommissionError):
qh.get_commission(self.client, s1+1)
r = qh.get_quota()
self.assertEqual(r,
{(holder, source, resource1): (limit1, 0, limit1/2),
(holder, source, resource2): (limit2, 0, limit2),
})
# commission exceptions
with assertRaises(NoCapacityError) as cm:
self.issue_commission([((holder, source, resource1), 1),
((holder, source, resource2), 1)])
e = cm.exception
provision = e.data['provision']
self.assertEqual(provision['holder'], holder)
self.assertEqual(provision['source'], source)
self.assertEqual(provision['resource'], resource2)
self.assertEqual(provision['quantity'], 1)
self.assertEqual(e.data['usage'], limit2)
self.assertEqual(e.data['limit'], limit2)
with assertRaises(NoQuantityError) as cm:
self.issue_commission([((holder, source, resource1), -1)])
e = cm.exception
provision = e.data['provision']
self.assertEqual(provision['holder'], holder)
self.assertEqual(provision['source'], source)
self.assertEqual(provision['resource'], resource1)
self.assertEqual(provision['quantity'], -1)
self.assertEqual(e.data['usage'], 0)
self.assertEqual(e.data['limit'], 0)
with assertRaises(NoHoldingError) as cm:
self.issue_commission([((holder, source, resource1), 1),
(('nonex', source, resource1), 1)])
e = cm.exception
provision = e.data['provision']
self.assertEqual(provision['holder'], 'nonex')
self.assertEqual(provision['source'], source)
self.assertEqual(provision['resource'], resource1)
self.assertEqual(provision['quantity'], 1)
r = qh.get_quota(holders=[holder])
quotas = {(holder, source, resource1): (limit1, 0, limit1/2),
(holder, source, resource2): (limit2, 0, limit2),
}
self.assertEqual(r, quotas)
# resolve commission
r = qh.get_pending_commissions(clientkey=self.client)
self.assertEqual(len(r), 1)
serial = r[0]
self.assertEqual(serial, s1)
r = qh.resolve_pending_commission(self.client, serial)
self.assertEqual(r, True)
r = qh.get_pending_commissions(clientkey=self.client)
self.assertEqual(r, [])
r = qh.resolve_pending_commission(self.client, serial)
self.assertEqual(r, False)
r = qh.get_quota(holders=[holder])
quotas = {(holder, source, resource1): (limit1, limit1/2, limit1/2),
(holder, source, resource2): (limit2, limit2, limit2),
}
self.assertEqual(r, quotas)
logs = models.ProvisionLog.objects.filter(serial=serial)
self.assertEqual(len(logs), 2)
log1 = filter(lambda p: p.resource == resource1
and p.delta_quantity == limit1/2
and p.usage_min == limit1/2, logs)
self.assertEqual(len(log1), 1)
# resolve commissions
serial = self.issue_commission([((holder, source, resource1), 1),
((holder, source, resource2), -1)])
r = qh.get_quota(holders=[holder])
quotas = {(holder, source, resource1): (limit1, limit1/2, limit1/2+1),
(holder, source, resource2): (limit2, limit2-1, limit2),
}
self.assertEqual(r, quotas)
r = qh.resolve_pending_commission(self.client, serial, accept=False)
self.assertEqual(r, True)
serial1 = self.issue_commission([((holder, source, resource1), 1),
((holder, source, resource2), -1)])
serial2 = self.issue_commission([((holder, source, resource1), 1),
((holder, source, resource2), -1)])
serial3 = self.issue_commission([((holder, source, resource1), 1),
((holder, source, resource2), -1)])
r = qh.resolve_pending_commissions(clientkey=self.client,
accept_set=[serial1, serial2, 0],
reject_set=[serial2, serial3])
self.assertEqual(r, ([serial1], [serial3], [0], [serial2]))
r = qh.get_pending_commissions(clientkey=self.client)
self.assertEqual(r, [serial2])
# forced commission
r = qh.get_quota(holders=[holder])
quotas = {
(holder, source, resource1): (limit1, limit1/2+1, limit1/2+2),
(holder, source, resource2): (limit2, limit2-2, limit2-1),
}
self.assertEqual(r, quotas)
with assertRaises(NoCapacityError):
self.issue_commission(
[((holder, source, resource1), limit1/2+1)])
serial = self.issue_commission(
[((holder, source, resource1), limit1/2+1)],
force=True)
r = qh.resolve_pending_commission(self.client, serial, accept=True)
self.assertEqual(r, True)
r = qh.get_quota(holders=[holder])
quotas = {
(holder, source, resource1): (limit1, limit1+2, limit1+3),
(holder, source, resource2): (limit2, limit2-2, limit2-1),
}
self.assertEqual(r, quotas)
with assertRaises(NoQuantityError):
self.issue_commission(
[((holder, source, resource1), -2*limit1)],
force=True)
# release off upper limit
serial = self.issue_commission([((holder, source, resource1), -1)])
r = qh.resolve_pending_commission(self.client, serial)
self.assertEqual(r, True)
r = qh.get_quota(holders=[holder], resources=[resource1])
quotas = {
(holder, source, resource1): (limit1, limit1+1, limit1+2),
}
self.assertEqual(r, quotas)
def test_020_empty_provisions(self):
serial = self.issue_commission([])
r = qh.resolve_pending_commission(self.client, serial)
self.assertEqual(r, True)
def handle(self, *args, **options):
write = self.stderr.write
force = options['force']
userid = options['userid']
project = options['project']
resources = [quotas.PENDING_APP_RESOURCE]
try:
astakos = Component.objects.get(name="astakos")
except Component.DoesNotExist:
raise CommandError("Component 'astakos' not found.")
query = [userid] if userid is not None else None
qh_holdings = quotas.service_get_quotas(astakos, query)
query = [project] if project is not None else None
qh_project_holdings = quotas.service_get_project_quotas(astakos, query)
if userid is None:
users = AstakosUser.objects.accepted().select_related(
'base_project')
else:
try:
user = AstakosUser.objects.get(uuid=userid)
except AstakosUser.DoesNotExist:
raise CommandError("There is no user with uuid '%s'." % userid)
if not user.is_accepted():
raise CommandError("%s is not an accepted user." % userid)
users = [user]
db_holdings = count_pending_app(users)
db_project_holdings = {}
for user, user_holdings in db_holdings.iteritems():
db_project_holdings.update(user_holdings)
unsynced_users, users_pending, users_unknown =\
reconcile.check_users(self.stderr, resources,
db_holdings, qh_holdings)
unsynced_projects, projects_pending, projects_unknown =\
reconcile.check_projects(self.stderr, resources,
db_project_holdings, qh_project_holdings)
pending_exists = users_pending or projects_pending
unknown_exists = users_unknown or projects_unknown
headers = ("Type", "Holder", "Source", "Resource", "Astakos",
"Quotaholder")
unsynced = unsynced_users + unsynced_projects
if unsynced:
pprint_table(self.stdout, unsynced, headers)
if options["fix"]:
user_provisions = create_user_provisions(unsynced_users)
project_provisions = create_project_provisions(
unsynced_projects)
provisions = user_provisions + project_provisions
name = ("client: reconcile-resources-astakos, time: %s" %
datetime.now())
try:
s = qh.issue_commission('astakos',
provisions,
name=name,
force=force)
except qh_exception.NoCapacityError:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
return
qh.resolve_pending_commission('astakos', s)
write("Fixed unsynced resources\n")
if pending_exists:
write("Found pending commissions. "
"This is probably a bug. Please report.\n")
elif not (unsynced or unknown_exists):
write("Everything in sync.\n")
def handle(self, *args, **options):
write = self.stderr.write
force = options['force']
userid = options['userid']
project = options['project']
resources = [quotas.PENDING_APP_RESOURCE]
try:
astakos = Component.objects.get(name="astakos")
except Component.DoesNotExist:
raise CommandError("Component 'astakos' not found.")
query = [userid] if userid is not None else None
qh_holdings = quotas.service_get_quotas(astakos, query)
query = [project] if project is not None else None
qh_project_holdings = quotas.service_get_project_quotas(astakos, query)
if userid is None:
users = AstakosUser.objects.accepted().select_related(
'base_project')
else:
try:
user = AstakosUser.objects.get(uuid=userid)
except AstakosUser.DoesNotExist:
raise CommandError("There is no user with uuid '%s'." % userid)
if not user.is_accepted():
raise CommandError("%s is not an accepted user." % userid)
users = [user]
db_holdings = count_pending_app(users)
db_project_holdings = {}
for user, user_holdings in db_holdings.iteritems():
db_project_holdings.update(user_holdings)
unsynced_users, users_pending, users_unknown =\
reconcile.check_users(self.stderr, resources,
db_holdings, qh_holdings)
unsynced_projects, projects_pending, projects_unknown =\
reconcile.check_projects(self.stderr, resources,
db_project_holdings, qh_project_holdings)
pending_exists = users_pending or projects_pending
unknown_exists = users_unknown or projects_unknown
headers = ("Type", "Holder", "Source", "Resource",
"Astakos", "Quotaholder")
unsynced = unsynced_users + unsynced_projects
if unsynced:
pprint_table(self.stdout, unsynced, headers)
if options["fix"]:
user_provisions = create_user_provisions(unsynced_users)
project_provisions = create_project_provisions(
unsynced_projects)
provisions = user_provisions + project_provisions
name = ("client: reconcile-resources-astakos, time: %s"
% datetime.now())
try:
s = qh.issue_commission('astakos', provisions,
name=name, force=force)
except qh_exception.NoCapacityError:
write("Reconciling failed because a limit has been "
"reached. Use --force to ignore the check.\n")
return
qh.resolve_pending_commission('astakos', s)
write("Fixed unsynced resources\n")
if pending_exists:
write("Found pending commissions. "
"This is probably a bug. Please report.\n")
elif not (unsynced or unknown_exists):
write("Everything in sync.\n")