def get_context(self, base_context=None):
context = base_context or {}
if self.submission:
has_documents = any([
subdoc.get("type", {}).get("key") == "respondent"
for subdoc in self.submission["documents"]
])
context["primary"] = (self.submission.get(
"deficiency_notice_params", {}).get("assign_user",
{}).get("contact_status"))
context["is_primary"] = context["primary"]
context["assign_user"] = self.submission["contact"]["user"]
context["representing"] = self.submission["organisation"]
context["representing_third_party"] = get(
self.submission, "organisation/id") != get(
self.submission, "contact/organisation/id")
context["has_documents"] = has_documents
context["enable_submit"] = context["primary"] and (
context["representing"]["id"] == self.user.organisation["id"]
or has_documents)
context["organisation"] = context.get("current_organisation")
if self.view:
documents = self.view.get_submission_documents(
request_for_sub_org=True)
context["all_documents"] = documents
context["documents"] = documents.get("caseworker", [])
return context
def get(self, request, organisation_id, *args, **kwargs):
org_matches = self.client(self.request.user).get_organisation_matches(
organisation_id, with_details="none")
org_index = deep_index_items_by(org_matches, "id")
for match in org_matches:
match_id = get(match, "id")
no_merge = get(match, "json_data/no_merge") or {}
if str(match_id) == str(organisation_id):
# this is our org, so remove everything from the no-merge list
for nm_id, details in no_merge.items():
if str(nm_id) in org_index:
del org_index[str(nm_id)]
else:
# This is not our org, so remove it if our org id is in its no-merge list
for nm_id, details in no_merge.items():
if str(nm_id) == str(organisation_id) and str(
match_id) in org_index:
del org_index[str(match_id)]
return HttpResponse(
json.dumps({
"org_matches":
[org_array[0] for org_array in org_index.values()],
}),
content_type="application/json",
)
def post(self,
request,
code=None,
case_id=None,
*args,
**kwargs): # noqa: C901
self.default_session(request)
redirect_postfix = f"{code}/{case_id}/" if code and case_id else ""
request.session["registration"].update(request.POST.dict())
errors = validate(request.session["registration"],
registration_validators)
if request.session["registration"].get("password") != request.session[
"registration"].get("password_confirm"):
errors["password_confirm"] = "Passwords do not match"
if not request.session["registration"].get("email"):
errors["email"] = "Email is required"
if not errors:
session_reg = request.session.get("registration", {})
if (session_reg.get("code") and session_reg.get("case_id")
and session_reg.get("confirm_invited_org") is True):
invitee_sec_group = get(request.session["registration"],
"invite/organisation_security_group")
if invitee_sec_group == SECURITY_GROUP_THIRD_PARTY_USER:
# Use the third party invitee's organisation
organisation_id = get(request.session["registration"],
"invite/contact/organisation/id")
else:
organisation_id = get(request.session["registration"],
"invite/organisation/id")
organisation = self.trusted_client.get_organisation(
organisation_id=organisation_id)
field_map = {
"id": "organisation_id",
"name": "organisation_name",
"address": "organisation_address",
}
out = {}
organisation_country_code = get(organisation, "country/code")
if organisation_country_code:
out["organisation_country_code"] = organisation_country_code
out["uk_company"] = "yes" if organisation_country_code == "GB" else "no"
for field, value in organisation.items():
out_field = field_map.get(field) or field
out[out_field] = value
self.update_session(request, out)
if organisation_country_code:
return redirect("/accounts/register/3/")
return redirect("/accounts/register/2/")
elif (session_reg.get("code") and session_reg.get("case_id")
and not session_reg.get("confirm_invited_org")):
return redirect(f"/accounts/register/2/{redirect_postfix}")
return redirect("/accounts/register/2/")
return redirect(f"/accounts/register/{redirect_postfix}?error")
def fit(self, X, Y, batch_size=50, epochs=1, clip=False):
Xtmp, Ytmp = X.split(batch_size), Y.split(batch_size)
for _ in range(epochs):
self.progbar.__init__(len(Xtmp))
for x, y in zip(Xtmp, Ytmp):
#self.optimizer.zero_grad()
loss = self.loss(self.forward(x), y)
self.optimize(loss, clip)
new_loss = self.loss(self.forward(x).detach(), y)
self.progbar.add(1,
values=[("old", U.get(loss)),
("new", U.get(new_loss))])
def post(self, request, task_id=None):
task_id = task_id or request.POST.get("task_id")
data = pluck(
request.POST,
[
"id",
"name",
"description",
"due_date",
"content_type",
"model_id",
"model_key",
"assignee",
"assignee_id",
"case_id",
"priority",
"status",
"data",
"btn_value",
"data_estimate",
"data_remaining",
],
)
json_data = data.get("data") or {}
regex = r"^data_"
for param_key in request.POST:
matches = re.split(regex, param_key)
if len(matches) > 1:
sub_key = matches[1]
value = request.POST[param_key]
if value == "__remove":
if get(json_data, sub_key):
json_data.pop(sub_key)
else:
json_data[sub_key] = value
data["data"] = json.dumps(json_data)
if data.get("btn_value") == "save":
cache.set("lasttaskupdate", timezone.now().strftime("%Y-%m-%dT%H:%M:%S"), 60 * 60)
if not task_id:
# we need the object attachments to create task
result = self.client(request.user).create_update_task(
content_type=request.POST.get("content_type"),
model_id=request.POST.get("model_id"),
data=data,
)
else:
result = self.client(request.user).create_update_task(task_id=task_id, data=data)
elif get(data, "btn_value") == "delete":
result = self.client(request.user).delete_task(task_id=get(data, "id"))
return HttpResponse(json.dumps({"result": result}), content_type="application/json")
def fit(self,X,Y,batch_size=50,epochs=1,clip=False,l1_decay=0.0):
if len(X)>batch_size:
Xtmp,Ytmp = X.split(batch_size),Y.split(batch_size)
else:
Xtmp,Ytmp=[X],[Y]
for _ in range(epochs):
if self.verbose:self.progbar.__init__(len(Xtmp))
for x,y in zip(Xtmp,Ytmp):
#self.optimizer.zero_grad()
loss = self.loss(self.forward(x),y)+l1_decay*self.l1_weight()
self.optimize(loss,clip)
new_loss = self.loss(self.forward(x).detach(),y)
self.progbar.add(1,values=[("old",U.get(loss)),("new",U.get(new_loss))])
return np.mean(self.progbar._values["new"])
def update(self, y):
self.Sig = (((1 - self.alpha) * self.Sig * y.shape[0] + self.alpha *
(y.t().mm(y))) / y.shape[0]).detach()
self.min_eig.append(float(U.get(self.Sig.eig()[0][:, 0].min())))
if self.min_eig[-1] < 0:
print("None SPD matrix in Cholesky")
self.update_weight()
def post(
self,
request,
case_id=None,
organisation_id=None,
organisation_type=None,
*args,
**kwargs,
):
extra_context = {"case_id": str(case_id)}
org_request_fields = collect_request_fields(request, org_fields,
extra_context)
if organisation_id:
organisation = self._client.update_organisation(
organisation_id, org_request_fields)
else:
# create new org and add to case
organisation = get(
self._client.update_case_organisation_by_type(
case_id, organisation_type, org_request_fields),
"organisation",
)
if not org_request_fields.get("organisation_id") and organisation:
contact_request_fields = collect_request_fields(
request, contact_fields, extra_context)
contact_request_fields["organisation_id"] = organisation["id"]
if contact_request_fields.get(
"contact_name") and contact_request_fields.get(
"contact_email"):
contact = self._client.create_contact(
contact_request_fields)
return self.return_redirect("Created party " +
organisation.get("name"))
return HttpResponse(json.dumps({"organisation": organisation}))
async def load_state():
try:
with open("saved_state.json", "r") as f:
data = json.loads(f.read())
except IOError:
return
log.info("Loading state...")
bot.allow_offline = list(data['allow_offline'])
for qd in data['queues']:
if qc := queue_channels.get(qd['channel_id']):
if q := get(qc.queues, id=qd['queue_id']):
await q.from_json(qd)
else:
log.error(f"Queue with id {qd['queue_id']} not found.")
def prob_predict(self,x):
self.eval()
x = U.torchify(x)
if len(x.shape) ==len(self.input_shape):
x.unsqueeze_(0)
y = U.get(self.logsoftmax(x).squeeze().exp())
self.train()
return y
def collate_cases(user_cases):
# flatten user_cases by org and case.
# so <org_id>/<case_id>/ contains {case:<case>,users:[<user>,..]}
nonlocal exclude_case_id
out = {}
for user_case in user_cases or []:
case_id = get(user_case, "case/id")
org_id = get(user_case, "organisation/id")
if str(case_id) != str(exclude_case_id):
case_org_id = f"{case_id}:{org_id}"
out.setdefault(
case_org_id,
{
"case": get(user_case, "case"),
"caserole": get(user_case, "caserole"),
"organisation": get(user_case, "organisation"),
"usercases": [],
},
)
if get(user_case, "caserole/validated_at"):
verified_cases[case_org_id] = out[case_org_id]
out[case_org_id]["usercases"].append(get(
user_case, "usercase"))
return list(out.values())
def log(self, key, val):
self.keys.add(key)
if type(val) == torch.Tensor:
self.log(key, U.get(val))
else:
if "numpy" in str(type(val)):
self.past[-1][key] = val.tolist()
else:
self.past[-1][key] = val
def predict(self,x):
self.eval()
x = U.torchify(x)
if len(x.shape) ==len(self.input_shape):
x.unsqueeze_(0)
y = U.get(self.forward(x).squeeze())
self.train()
return y
def train(self,states, options, actions, advantages, tdlamret, gate_losses):
n = (options==self.option_n).sum()
self.log("Selected",n)
if n >2:
kl_get = self.calculate_kl(states, options, actions, advantages)
loss_grad = self.policy.flaten.flatgrad(gate_losses["gain"], retain=True)
surrogate_before = kl_get(grad=True)
grad_kl = self.policy.flaten.flatgrad(surrogate_before, create=True, retain=True)
theta_before = self.policy.flaten.get()
self.log("Init param sum", theta_before.sum())
if np.allclose(U.get(loss_grad), 0, atol=1e-15):
C.warning("Got zero gradient. not updating %i"%self.option_n)
else:
stepdir = m_utils.conjugate_gradient(self.Fvp(grad_kl), loss_grad, cg_iters=self.cg_iters)
assert stepdir.sum()!=float("Inf")
shs = .5*stepdir.dot(self.Fvp(grad_kl)(stepdir))
lm = torch.sqrt(shs / self.max_kl)
self.log("lagrange multiplier:", lm)
self.log("gnorm:", np.linalg.norm(loss_grad.cpu().detach().numpy()))
fullstep = stepdir / lm
expected_improve = loss_grad.dot(fullstep)
with C.timeit("Line Search"):
stepsize = 1.0
for i in range(10):
theta_new = theta_before + fullstep * stepsize
self.policy.flaten.set(theta_new)
#losses = self.calculate_losses(states,actions,advantages)
surr = gate_losses["surr_get"]()
improve = surr - surrogate_before
kl = kl_get()
if surr == float("Inf") or kl ==float("Inf"):
C.warning("Infinite value of losses %i"%self.option_n)
elif kl > self.max_kl:
C.warning("Violated KL %i"%self.option_n)
elif improve < 0:
stepsize *= self.ls_step
else:
self.log("Line Search","OK")
break
else:
improve = 0
self.log("Line Search","NOPE")
self.policy.flaten.set(theta_before)
self.log("Expected",expected_improve)
self.log("Actual",improve)
self.log("LS Steps",i)
self.log("KL",kl)
loss = self.value_function.fit(states[options==self.option_n], tdlamret[options==self.option_n], batch_size = 32, epochs = self.vf_iters,l1_decay=1e-4)
self.log("Vfunction loss",loss)
del(kl_get, loss_grad, grad_kl,theta_before)
async def from_json(cls, queue, qc, data):
# Prepare discord objects
data['players'] = [
qc.channel.guild.get_member(user_id) for user_id in data['players']
]
if None in data['players']:
await qc.error(
f"Unable to load match {data['match_id']}, error fetching guild members."
)
return
# Fill data with discord objects
for i in range(len(data['teams'])):
data['teams'][i] = [
get(data['players'], id=user_id)
for user_id in data['teams'][i]
]
data['ready_players'] = [
get(data['players'], id=user_id)
for user_id in data['ready_players']
]
# Create the Match object
ratings = {
p['user_id']: p['rating']
for p in await qc.rating.get_players((p.id
for p in data['players']))
}
bot.last_match_id += 1
match = cls(bot.last_match_id, queue, qc, data['players'], ratings,
**data['cfg'])
# Set state data
for i in range(len(match.teams)):
match.teams[i].set(data['teams'][i])
match.check_in.ready_players = set(data['ready_players'])
match.maps = data['maps']
match.state = data['state']
match.states = data['states']
if match.state == match.CHECK_IN:
await match.check_in.start() # Spawn a new check_in message
bot.active_matches.append(match)
def transform(self, state0):
state = state0[self.crop['up']:self.crop['down'],
self.crop['left']:self.crop['right'], :].astype(int)
state = skimage.transform.resize(state.astype(float),
(self.size, self.size, 3),
mode="constant") / 255.0
self.last_frame = state.copy()
self.episode.append(self.last_frame)
state = state.astype(float).transpose(self.axis)
return U.get(self.spin(U.torchify(state).unsqueeze(0))).squeeze()
def add_contact(case_contact, case):
if get(case_contact, "contact").address != "redacted":
contact_id = str(get(case_contact, "contact").id)
if contact_id not in contacts:
contact = get(case_contact, "contact")
contacts[contact_id] = contact.to_dict(case)
contacts[contact_id]["cases"] = {}
if get(contacts[contact_id], "user"):
try:
organisation = contact.user.organisation.organisation
contacts[contact_id]["user"]["organisation"] = {
"id": organisation.id
}
except AttributeError:
pass
if case_contact.get("loa"):
contacts[contact_id]["loa"] = True
if case_contact.get("organisation_contact"):
contacts[contact_id]["organisation_contact"] = True
if case_contact.get("case_contact"):
contacts[contact_id]["case_contact"] = True
case = get(case_contact, "case")
if case:
cc = contacts[contact_id]["cases"].get(str(case.id)) or {
"name": case.name
}
cc.update({
k: v
for k, v in case_contact.items()
if v is not None and k in
["primary", "loa", "name", "in_case", "case_contact"]
})
contacts[contact_id]["cases"][str(case.id)] = cc
def on_submit(self, **kwargs):
"""
Triggered when user assignment is submitted. If the assignment is to own case,
the user will be assigned.
Returns an overriden redirect url if the assignment was successful.
"""
user_organisation_id = (get(self.submission, "contact/organisation/id")
or get(self.submission,
"contact/user/organisation/id")
or get(self.user.organisation, "id"))
user_id = get(self.submission, "contact/user/id")
if get(self.submission, "organisation/id") == user_organisation_id:
# make the case assignment.
is_primary = (self.submission.get(
"deficiency_notice_params",
{}).get("assign_user", {}).get("contact_status") == "primary")
self.client.assign_user_to_case(
user_organisation_id=user_organisation_id,
representing_id=get(self.submission, "organisation/id"),
user_id=user_id,
case_id=self.case["id"],
primary=is_primary,
)
self.client.set_submission_state(self.case["id"],
self.submission["id"],
"sufficient")
return f"/accounts/team/{user_organisation_id}/user/{user_id}/?alert=user-assigned"
return f"/accounts/team/{user_organisation_id}/user/{user_id}/?alert=user-assigned-req"
def get(self, request, *args, **kwargs):
organisations = self._client.get_organisations(fields=self.fields)
organisations.sort(key=lambda org: get(org, "name", "").lower())
groups = {"active": [], "inactive": [], "fraudulent": []}
for organisation in organisations:
if organisation.get("fraudulent"):
groups["fraudulent"].append(organisation)
elif not organisation.get("case_count"):
groups["inactive"].append(organisation)
else:
groups["active"].append(organisation)
tab = request.GET.get("tab") or "active"
tabs = {
"value":
tab,
"tabList": [
{
"label": "Active organisations",
"value": "active",
"sr_text": "Organisations that have case participations",
},
{
"label":
"Fraudulent",
"value":
"fraudulent",
"sr_text":
"Organisations that have been marrked as fraudulent",
},
{
"label":
"No case participation",
"value":
"inactive",
"sr_text":
"Organisations tha have never participated in a case",
},
],
}
for tab_iter in tabs.get("tabList"):
tab_iter["count"] = len(groups.get(tab_iter["value"]) or [])
return render(
request,
self.template_name,
{
"tabs": tabs,
"body_classes": "full-width",
"organisations": groups.get(tab),
},
)
def save(self, *args, **kwargs):
"""
Inject next task reference number
"""
if not self.reference:
self.reference = (
get(
Task.objects.filter(case_id=self.case_id).aggregate(
ref=models.Max("reference")
),
"ref",
)
or 0
) + 1
return super().save(*args, **kwargs)
def decorate_orgs(org_list, representing_org_id, exclude_case_id=None):
# If exclude_case_id is provided, references to that case are excluded
verified_cases = {}
def collate_cases(user_cases):
# flatten user_cases by org and case.
# so <org_id>/<case_id>/ contains {case:<case>,users:[<user>,..]}
nonlocal exclude_case_id
out = {}
for user_case in user_cases or []:
case_id = get(user_case, "case/id")
org_id = get(user_case, "organisation/id")
if str(case_id) != str(exclude_case_id):
case_org_id = f"{case_id}:{org_id}"
out.setdefault(
case_org_id,
{
"case": get(user_case, "case"),
"caserole": get(user_case, "caserole"),
"organisation": get(user_case, "organisation"),
"usercases": [],
},
)
if get(user_case, "caserole/validated_at"):
verified_cases[case_org_id] = out[case_org_id]
out[case_org_id]["usercases"].append(get(
user_case, "usercase"))
return list(out.values())
for org in org_list:
org["collated_cases"] = collate_cases(org.get("cases"))
org["collated_indirect_cases"] = collate_cases(
org.get("indirect_cases"))
usercases = org.get("collated_indirect_cases")
if usercases:
org["cases_for_org"] = list([
case for case in usercases
if get(case, "organisation/id") == representing_org_id
])
org["verified_usercases"] = list(verified_cases.values())
org["verified_usercases"].sort(
key=lambda uc: get(uc, "caserole/validated_at", ""), reverse=True)
return org_list
def post(self, request, code=None, case_id=None, *args, **kwargs):
redirect_postfix = f"{code}/{case_id}/" if code and case_id else ""
if "registration" not in request.session:
return redirect(f"/accounts/register/{redirect_postfix}")
self.update_session(request, request.POST.dict())
request.session["registration"].pop("errors", None) # Clear existing
errors = validate(request.session["registration"],
self.validators) or {}
if get(request.session["registration"], "uk_company") == "no":
errors.update(
validate(request.session["registration"],
self.country_validator) or {})
if not errors:
next_page = ("5" if request.session["registration"].get(
"same_contact_address") == "yes" else "4")
return redirect(
f"/accounts/register/{next_page}/{redirect_postfix}")
else:
request.session["registration"]["errors"] = errors
request.session.modified = True
return redirect(f"/accounts/register/3/{redirect_postfix}")
def train(self):
self.progbar.__init__(self.memory_min)
while (self.memory.size < self.memory_min):
self.path_generator.__next__()
while (self.done < self.train_steps):
to_log = 0
self.progbar.__init__(self.update_double)
old_theta = self.Q.flaten.get()
self.target_Q.copy(self.Q)
while to_log < self.update_double:
self.path_generator.__next__()
rollout = self.memory.sample(self.batch_size)
state_batch = U.torchify(rollout["state"])
action_batch = U.torchify(rollout["action"]).long()
reward_batch = U.torchify(rollout["reward"])
non_final_batch = U.torchify(1 - rollout["terminated"])
next_state_batch = U.torchify(rollout["next_state"])
#current_q = self.Q(state_batch)
current_q = self.Q(state_batch).gather(
1, action_batch.unsqueeze(1)).view(-1)
_, a_prime = self.Q(next_state_batch).max(1)
# Compute the target of the current Q values
next_max_q = self.target_Q(next_state_batch).gather(
1, a_prime.unsqueeze(1)).view(-1)
target_q = reward_batch + self.discount * non_final_batch * next_max_q.squeeze(
)
# Compute loss
loss = self.Q.loss(current_q, target_q.detach(
)) # loss = self.Q.total_loss(current_q, target_q)
# Optimize the model
self.Q.optimize(loss, clip=True)
self.progbar.add(self.batch_size,
values=[("Loss", U.get(loss))])
to_log += self.batch_size
self.target_Q.copy(self.Q)
new_theta = self.Q.flaten.get()
self.log("Delta Theta L1",
U.get((new_theta - old_theta).abs().mean()))
self.log("Av 50ep rew", np.mean(self.past_rewards))
self.log("Max 50ep rew", np.max(self.past_rewards))
self.log("Min 50ep rew", np.min(self.past_rewards))
self.log("Epsilon", self.eps)
self.log("Done", self.done)
self.log("Total", self.train_steps)
self.target_Q.copy(self.Q)
self.print()
#self.play()
self.save()
log.info("Loading state...")
bot.allow_offline = list(data['allow_offline'])
for qd in data['queues']:
if qc := queue_channels.get(qd['channel_id']):
if q := get(qc.queues, id=qd['queue_id']):
await q.from_json(qd)
else:
log.error(f"Queue with id {qd['queue_id']} not found.")
else:
log.error(f"Queue channel with id {qd['channel_id']} not found.")
for md in data['matches']:
if qc := queue_channels.get(md['channel_id']):
if q := get(qc.queues, id=md['queue_id']):
await bot.Match.from_json(q, qc, md)
else:
log.error(f"Queue with id {md['queue_id']} not found.")
else:
log.error(f"Queue channel with id {md['channel_id']} not found.")
if 'expire' in data.keys():
await bot.expire.load_json(data['expire'])
async def remove_players(*users, reason=None):
for qc in set((q.qc for q in active_queues)):
await qc.remove_members(*users, reason=reason)
def predict(self, x):
x = U.torchify(x)
if len(x.shape) == len(self.input_shape):
x.unsqueeze_(0)
return U.get(self.forward(x).squeeze())
def _get(arg1, arg2):
"""get a value from an object"""
return get(arg1, str(arg2))
