def _update_user_membership(self, tas_project, username, action=None):
if action not in ["add", "delete"]:
raise ValueError("Invalid membership action {}".format(action))
charge_code = self.get_attr(tas_project, "chargeCode")
keycloak_client = KeycloakClient()
keycloak_client.update_membership(charge_code, username, action)
def save_project(self, proj, host=None):
allocations = self.get_attr(proj, "allocations")
reformated_proj = self.tas_to_portal_proj_obj(proj)
reformated_proj.save()
if reformated_proj.charge_code.startswith(
TMP_PROJECT_CHARGE_CODE_PREFIX):
# save project in portal
new_proj = portal_proj.objects.filter(
charge_code=reformated_proj.charge_code)
if len(new_proj) == 0:
logger.error(
f"Couldn't find project {reformated_proj.charge_code}")
else:
new_proj = new_proj[0]
valid_charge_code = ("CHI-" + str(datetime.today().year)[2:] +
str(new_proj.id).zfill(4))
new_proj.charge_code = valid_charge_code
new_proj.save()
reformated_proj.charge_code = valid_charge_code
# create allocation
self.save_allocation(allocations[0], valid_charge_code, host)
# save project in keycloak
keycloak_client = KeycloakClient()
keycloak_client.create_project(valid_charge_code,
new_proj.pi.username)
return self.portal_to_tas_proj_obj(reformated_proj,
fetch_allocations=False)
def end_daypasses():
beyond_duration_invitations = get_invitations_beyond_duration()
for invitation in beyond_duration_invitations:
try:
LOG.info(f"Removing user from project with invite {invitation.id}\n")
project = Project.objects.get(pk=invitation.project_id)
user = User.objects.get(pk=invitation.user_accepted_id)
keycloak_client = KeycloakClient()
keycloak_client.update_membership(
project.charge_code, user.username, "delete"
)
invitation.status = Invitation.STATUS_BEYOND_DURATION
invitation.save()
try:
daypass_request = DaypassRequest.objects.get(invitation=invitation)
approved_requests = (
DaypassRequest.objects.all()
.filter(
artifact=daypass_request.artifact,
status=DaypassRequest.STATUS_APPROVED,
invitation__status=Invitation.STATUS_BEYOND_DURATION,
)
.count()
)
if approved_requests == settings.DAYPASS_LIMIT:
# Send an email
handle_too_many_daypass_users(daypass_request.artifact)
except DaypassRequest.DoesNotExist:
pass
except Exception as e:
LOG.error(f"Error ending daypass invite {invitation.id}: {e}")
def get_all_alloc(request):
"""Get all allocations, grouped by project.
Args:
request: the request that is passed in.
Raises:
Exception: when loading projects fails.
Returns:
json: dumps all data as serialized json.
"""
try:
keycloak_client = KeycloakClient()
user_attributes = keycloak_client.get_all_users_attributes()
mapper = ProjectAllocationMapper(request)
resp = mapper.get_all_projects()
logger.debug("Total projects: %s", len(resp))
for r in resp:
pi_attributes = user_attributes.get(r["pi"]["username"], {})
if pi_attributes:
institution = pi_attributes.get("affiliationInstitution", [])
country = pi_attributes.get("country", [])
r["pi"]["institution"] = next(iter(institution), None)
r["pi"]["country"] = next(iter(country), None)
except Exception as e:
logger.exception("Error loading chameleon projects")
messages.error(request, e)
raise
return json.dumps(resp)
def update_user_profile(self, user, new_profile,
is_request_pi_eligibililty):
keycloak_client = KeycloakClient()
if is_request_pi_eligibililty:
pie_request = PIEligibility()
pie_request.requestor_id = user.id
pie_request.save()
self._create_ticket_for_pi_request(user)
email = new_profile.get("email")
keycloak_client.update_user(
user.username,
email=email,
affiliation_title=new_profile.get("title"),
affiliation_department=new_profile.get("department"),
affiliation_institution=new_profile.get("institution"),
country=new_profile.get("country"),
citizenship=new_profile.get("citizenship"),
phone=new_profile.get("phone"),
)
# The email normally is saved during login; in this case we can
# immediately persist the change for better UX.
if email is not None:
user.email = email
user.save()
def add_openstack_data(self, **kwargs):
ticket_id = kwargs.get("ticket_id")
username = kwargs.get("username")
messages = []
bound_task = self
def write_message(progress_pct, message):
LOG.info(message)
messages.append(message)
bound_task.update_state(
state="PROGRESS",
meta={
"messages": messages,
"progress_pct": progress_pct,
},
)
try:
if username:
keycloak_client = KeycloakClient()
projects = keycloak_client.get_full_user_projects_by_username(
username)
regions = list(settings.OPENSTACK_AUTH_REGIONS.keys())
region_list = []
for i, region in enumerate(regions):
try:
factor = (1.0 / len(regions)) * 100
write_message(factor * i, f'Processing region "{region}"')
region_list.append(
get_openstack_data(username, region, projects))
except Exception as err:
LOG.error(
f"Failed to get OpenStack data for region {region}: {err}"
)
openstack_user_data = remove_empty_lines(
render_to_string("djangoRT/project_details.txt",
{"regions": region_list}))
else:
openstack_user_data = "No openstack data for anonymous user."
rt = rtUtil.DjangoRt()
rt.commentOnTicket(ticket_id, openstack_user_data)
except Exception as exc:
LOG.exception("Failed to gather data")
exc_message = getattr(exc, "message", None)
if exc_message:
messages.append(exc_message)
raise OpenstackDataError(messages=messages) from exc
# Return current state as last action
return {
"messages": messages,
"progress_pct": 100.0,
}
def get_project_members(project):
users = []
# try get members from keycloak
keycloak_client = KeycloakClient()
for username in keycloak_client.get_project_members(
get_charge_code(project)):
try:
user = get_user_model().objects.get(username=username)
users.append(user)
except get_user_model().DoesNotExist:
logger.exception(f"Could not get user model for {username}")
return users
def _update_user_membership(self, tas_project, user_ref, action=None):
if action not in ["add", "delete"]:
raise ValueError("Invalid membership action {}".format(action))
UserModel = get_user_model()
try:
user = UserModel.objects.get(username=user_ref)
except UserModel.DoesNotExist:
user = UserModel.objects.get(email=user_ref)
charge_code = self.get_attr(tas_project, "chargeCode")
keycloak_client = KeycloakClient()
keycloak_client.update_membership(charge_code, user.username, action)
def get_project_members(self, tas_project):
users = []
# try get members from keycloak
keycloak_client = KeycloakClient()
pi_username = tas_project.pi.username
for username in keycloak_client.get_project_members(
tas_project.chargeCode):
if username == pi_username:
role = "PI"
else:
role = "Standard"
user = self.get_user(username, to_pytas_model=True, role=role)
if user:
users.append(user)
return users
def active_approved_allocations(balance_service):
now = datetime.now(pytz.utc)
approved_allocations = Allocation.objects.filter(status='approved',
start_date__lte=now)
activated_alloc_count = 0
for alloc in approved_allocations:
charge_code = alloc.project.charge_code
# deactivate active allocation for the project and set status to active
project_active_allocations = Allocation.objects.filter(
status='active', project_id=alloc.project.id)
prev_alloc = None
if len(project_active_allocations) > 0:
prev_alloc = project_active_allocations[0]
try:
with transaction.atomic():
if prev_alloc:
_deactivate_allocation(balance_service, prev_alloc)
alloc.status = 'active'
alloc.save()
KeycloakClient().update_project(charge_code,
has_active_allocation='true')
# recharge balance service
balance_service.recharge(charge_code, alloc.su_allocated)
activated_alloc_count = activated_alloc_count + 1
except Exception:
logger.exception(f'Error activating project {charge_code}')
logger.debug(
'need to activated {} allocations, and {} were actually activated'.
format(len(approved_allocations), activated_alloc_count))
def keycloak_metadata(self, obj):
"""User metadata from keycloak backend. Returns a list of strings."""
keycloak_client = KeycloakClient()
keycloak_user = keycloak_client.get_user_by_username(
obj.requestor.username)
if not keycloak_user:
return "No Keycloak User Found"
full_name = "{} {}".format(keycloak_user["lastName"],
keycloak_user["firstName"])
email = keycloak_user["email"]
yield [f"Name: {full_name}", f"Email: {email}"]
for key, val in keycloak_user["attributes"].items():
if key not in ["joinDate"]:
# convert camelcase to separate out words
key = re.sub("([A-Z])", " \\1", key).strip().capitalize()
yield [(f"{key}: {val}")]
def _deactivate_allocation(balance_service, alloc):
charge_code = alloc.project.charge_code
balance = balance_service.get_balance(charge_code) or {}
if 'used' in balance and balance['used']:
alloc.su_used = float(balance['used'])
else:
alloc.su_used = None
logger.error(f'Couldn\'t find used balance for project {charge_code}')
alloc.status = 'inactive'
alloc.save()
KeycloakClient().update_project(charge_code, has_active_allocation='false')
def get_user_projects(self,
username,
alloc_status=[],
fetch_balance=True,
to_pytas_model=False):
# get user projects from portal
keycloak_client = KeycloakClient()
charge_codes = keycloak_client.get_user_projects_by_username(username)
projects_qs = portal_proj.objects.filter(charge_code__in=charge_codes)
user_projects = [
self.portal_to_tas_proj_obj(p, alloc_status=alloc_status)
for p in self._with_relations(projects_qs,
fetch_balance=fetch_balance)
]
if to_pytas_model:
return [tas_proj(initial=p) for p in user_projects]
else:
return user_projects
def _deactivate_allocation(alloc):
balance = project_balances([alloc.project.id])
if not balance:
alloc.su_used = None
LOG.error(f"Couldn't find used balance for project {alloc.project.charge_code}")
else:
balance = balance[0]
alloc.su_used = balance["used"]
alloc.status = "inactive"
alloc.save()
KeycloakClient().update_project(
alloc.project.charge_code, has_active_allocation="false"
)
def update_user_metadata_from_keycloak(self, tas_formatted_user):
keycloak_client = KeycloakClient()
keycloak_user = keycloak_client.get_user_by_username(
tas_formatted_user["username"])
if keycloak_user:
attrs = keycloak_user["attributes"]
tas_formatted_user.update({
"institution":
attrs.get("affiliationInstitution", None),
"department":
attrs.get("affiliationDepartment", None),
"title":
attrs.get("affiliationTitle", None),
"country":
attrs.get("country", None),
"phone":
attrs.get("phone", None),
"citizenship":
attrs.get("citizenship", None),
})
return tas_formatted_user
def list_daypass_requests(request, **kwargs):
keycloak_client = KeycloakClient()
projects = [
project["groupName"]
for project in keycloak_client.get_user_roles(request.user.username)
if manage_membership_in_scope(project["scopes"])
]
pending_requests = (DaypassRequest.objects.all().filter(
artifact__project__charge_code__in=projects,
status=DaypassRequest.STATUS_PENDING,
).order_by("-created_at"))
for daypass_request in pending_requests:
daypass_request.url = reverse("sharing_portal:review_daypass",
args=[daypass_request.id])
reviewed_requests = (DaypassRequest.objects.all(
).exclude(status=DaypassRequest.STATUS_PENDING).filter(
artifact__project__charge_code__in=projects).order_by("-created_at"))
template = loader.get_template("sharing_portal/list_daypass_requests.html")
context = {
"pending_requests": pending_requests,
"reviewed_requests": reviewed_requests,
}
return HttpResponse(template.render(context, request))
def lazy_add_user_to_keycloak(self):
keycloak_client = KeycloakClient()
# check if user exist in keycloak
keycloak_user = keycloak_client.get_user_by_username(self.current_user)
if keycloak_user:
return
user = self.get_user(self.current_user)
portal_user = self._get_user_from_portal_db(self.current_user)
join_date = None
if portal_user:
join_date = datetime.timestamp(portal_user.date_joined)
kwargs = {
"first_name": user["firstName"],
"last_name": user["lastName"],
"email": user["email"],
"affiliation_title": user["title"],
"affiliation_department": user["department"],
"affiliation_institution": user["institution"],
"country": user["country"],
"citizenship": user["citizenship"],
"join_date": join_date,
}
keycloak_client.create_user(self.current_user, **kwargs)
def active_approved_allocations(balance_service):
now = timezone.now()
approved_allocations = Allocation.objects.filter(
status="approved", start_date__lte=now
)
activated_alloc_count = 0
for alloc in approved_allocations:
charge_code = alloc.project.charge_code
# deactivate active allocation for the project and set status to active
project_active_allocations = Allocation.objects.filter(
status="active", project_id=alloc.project.id
)
prev_alloc = None
if len(project_active_allocations) > 0:
prev_alloc = project_active_allocations[0]
try:
with transaction.atomic():
if prev_alloc:
_deactivate_allocation(prev_alloc)
allocation_charges = Charge.objects.filter(
allocation__id=prev_alloc.id
)
# duplicate the ongoing charges
for c in allocation_charges:
if c.end_time > now:
_fork_charge(c, now, alloc)
alloc.status = "active"
alloc.save()
# TODO: remove recharge external balance service
# after retiring redis
balance_service.recharge(charge_code, alloc.su_allocated)
KeycloakClient().update_project(
charge_code, has_active_allocation="true"
)
activated_alloc_count = activated_alloc_count + 1
except Exception:
LOG.exception(f'Error activating project {charge_code}')
LOG.info(f'Started allocation {alloc.id} for {charge_code}')
LOG.debug('need to activated {} allocations, and {} were actually activated'.format(len(approved_allocations), activated_alloc_count))
def view_project(request, project_id):
mapper = ProjectAllocationMapper(request)
keycloak_client = KeycloakClient()
try:
project = mapper.get_project(project_id)
if project.source != "Chameleon":
raise Http404("The requested project does not exist!")
except Exception as e:
logger.error(e)
raise Http404("The requested project does not exist!")
form = ProjectAddUserForm()
nickname_form = EditNicknameForm()
type_form_args = {"request": request}
type_form = EditTypeForm(**type_form_args)
pubs_form = AddBibtexPublicationForm()
can_manage_project_membership, can_manage_project = get_user_permissions(
keycloak_client, request.user.username, project)
if (request.POST and can_manage_project_membership
or is_admin_or_superuser(request.user)):
form = ProjectAddUserForm()
if "add_user" in request.POST:
form = ProjectAddUserForm(request.POST)
if form.is_valid():
try:
add_username = form.cleaned_data["user_ref"]
user = User.objects.get(username=add_username)
if mapper.add_user_to_project(project, add_username):
messages.success(
request,
f'User "{add_username}" added to project!')
form = ProjectAddUserForm()
except User.DoesNotExist:
# Try sending an invite
email_address = form.cleaned_data["user_ref"]
try:
validate_email(email_address)
if email_exists_on_project(project, email_address):
messages.error(
request,
"That email is tied to a user already on the "
"project!",
)
else:
add_project_invitation(
project_id,
email_address,
request.user,
request,
None,
)
messages.success(request, "Invite sent!")
except ValidationError:
messages.error(
request,
("Unable to add user. Confirm that the username "
"is correct and corresponds to a current "
"Chameleon user. You can also send an invite "
"to an email address if the user does not yet "
"have an account."),
)
except Exception:
messages.error(
request,
"Problem sending invite, please try again.")
except Exception:
logger.exception("Failed adding user")
messages.error(request,
"Unable to add user. Please try again.")
else:
messages.error(
request,
("There were errors processing your request. "
"Please see below for details."),
)
elif "del_user" in request.POST:
try:
del_username = request.POST["user_ref"]
# Ensure that it's not possible to remove the PI
if del_username in [project.pi.username, project.pi.email]:
raise PermissionDenied(
"Removing the PI from the project is not allowed.")
if mapper.remove_user_from_project(project, del_username):
messages.success(
request,
'User "%s" removed from project' % del_username)
user = User.objects.get(username=del_username)
daypass = get_daypass(user.id, project_id)
if daypass:
daypass.delete()
except PermissionDenied as exc:
messages.error(request, exc)
except Exception:
logger.exception("Failed removing user")
messages.error(
request,
"An unexpected error occurred while attempting "
"to remove this user. Please try again",
)
elif "change_role" in request.POST:
try:
role_username = request.POST["user_ref"]
role_name = request.POST["user_role"].lower()
keycloak_client.set_user_project_role(role_username,
get_charge_code(project),
role_name)
except Exception:
logger.exception("Failed to change user role")
messages.error(
request,
"An unexpected error occurred while attempting "
"to change role for this user. Please try again",
)
elif "del_invite" in request.POST:
try:
invite_id = request.POST["invite_id"]
remove_invitation(invite_id)
messages.success(request, "Invitation removed")
except Exception:
logger.exception("Failed to delete invitation")
messages.error(
request,
"An unexpected error occurred while attempting "
"to remove this invitation. Please try again",
)
elif "resend_invite" in request.POST:
try:
invite_id = request.POST["invite_id"]
resend_invitation(invite_id, request.user, request)
messages.success(request, "Invitation resent")
except Exception:
logger.exception("Failed to resend invitation")
messages.error(
request, "An unexpected error occurred while attempting "
"to resend this invitation. Please try again")
elif "nickname" in request.POST:
nickname_form = edit_nickname(request, project_id)
elif "typeId" in request.POST:
type_form = edit_type(request, project_id)
for a in project.allocations:
if a.start and isinstance(a.start, str):
a.start = datetime.strptime(a.start, "%Y-%m-%dT%H:%M:%SZ")
if a.dateRequested:
if isinstance(a.dateRequested, str):
a.dateRequested = datetime.strptime(a.dateRequested,
"%Y-%m-%dT%H:%M:%SZ")
if a.dateReviewed:
if isinstance(a.dateReviewed, str):
a.dateReviewed = datetime.strptime(a.dateReviewed,
"%Y-%m-%dT%H:%M:%SZ")
if a.end:
if isinstance(a.end, str):
a.end = datetime.strptime(a.end, "%Y-%m-%dT%H:%M:%SZ")
users = get_project_members(project)
if not project_member_or_admin_or_superuser(request.user, project, users):
raise PermissionDenied
user_roles = keycloak_client.get_roles_for_all_project_members(
get_charge_code(project))
users_mashup = []
for u in users:
if u.username == project.pi.username:
continue
u_role = user_roles.get(u.username, "member")
user = {
"id": u.id,
"username": u.username,
"role": u_role.title(),
}
try:
portal_user = User.objects.get(username=u.username)
user["email"] = portal_user.email
user["first_name"] = portal_user.first_name
user["last_name"] = portal_user.last_name
# Add if the user is on a daypass
existing_daypass = get_daypass(portal_user.id, project_id)
if existing_daypass:
user["daypass"] = format_timedelta(
existing_daypass.date_exceeds_duration() - timezone.now())
except User.DoesNotExist:
logger.info("user: "******" not found")
users_mashup.append(user)
invitations = Invitation.objects.filter(project=project_id)
invitations = [i for i in invitations if i.can_accept()]
clean_invitations = []
for i in invitations:
new_item = {}
new_item["email_address"] = i.email_address
new_item["id"] = i.id
new_item["status"] = i.status.title()
if i.duration:
new_item["duration"] = i.duration
clean_invitations.append(new_item)
is_on_daypass = get_daypass(request.user.id, project_id) is not None
return render(
request,
"projects/view_project.html",
{
"project": project,
"project_nickname": project.nickname,
"project_type": project.type,
"users": users_mashup,
"invitations": clean_invitations,
"can_manage_project_membership": can_manage_project_membership,
"can_manage_project": can_manage_project,
"is_admin": request.user.is_superuser,
"is_on_daypass": is_on_daypass,
"form": form,
"nickname_form": nickname_form,
"type_form": type_form,
"pubs_form": pubs_form,
"roles": ROLES,
"host": request.get_host(),
},
)
def create_project(request):
mapper = ProjectAllocationMapper(request)
form_args = {"request": request}
user = mapper.get_user(request.user.username)
if user["piEligibility"].lower() != "eligible":
messages.error(
request,
"Only PI Eligible users can create new projects. "
"If you would like to request PI Eligibility, please "
'<a href="/user/profile/edit/">submit a PI Eligibility '
"request</a>.",
)
return HttpResponseRedirect(reverse("projects:user_projects"))
if request.POST:
form = ProjectCreateForm(request.POST, **form_args)
allocation_form = AllocationCreateForm(
request.POST, initial={"publication_up_to_date": True})
allocation_form.fields[
"publication_up_to_date"].widget = forms.HiddenInput()
funding_formset = FundingFormset(request.POST, initial=[{}])
consent_form = ConsentForm(request.POST)
if (form.is_valid() and allocation_form.is_valid()
and funding_formset.is_valid() and consent_form.is_valid()):
# title, description, typeId, fieldId
project = form.cleaned_data.copy()
allocation_data = allocation_form.cleaned_data.copy()
# let's check that any provided nickname is unique
project["nickname"] = project["nickname"].strip()
nickname_valid = (project["nickname"]
and ProjectExtras.objects.filter(
nickname=project["nickname"]).count() < 1
and Project.objects.filter(
nickname=project["nickname"]).count() < 1)
if not nickname_valid:
form.add_error("__all__", "Project nickname unavailable")
return render(request, "projects/create_project.html",
{"form": form})
# pi
pi_user_id = mapper.get_portal_user_id(request.user.username)
project["piId"] = pi_user_id
# allocations
allocation = {
"resourceId": 39,
"requestorId": pi_user_id,
"computeRequested": 20000,
"justification": allocation_data.pop("justification", None),
}
project["allocations"] = [allocation]
project["description"] = allocation_data.pop("description", None)
# source
project["source"] = "Chameleon"
created_project = None
try:
with transaction.atomic():
created_project = mapper.save_project(
project, request.get_host())
_save_fundings(funding_formset, created_project["id"])
logger.info("newly created project: " +
json.dumps(created_project))
messages.success(request, "Your project has been created!")
return HttpResponseRedirect(
reverse("projects:view_project",
args=[created_project["id"]]))
except:
# delete project from keycloak
if created_project:
keycloak_client = KeycloakClient()
keycloak_client.delete_project(
created_project["chargeCode"])
logger.exception("Error creating project")
form.add_error(
"__all__",
"An unexpected error occurred. Please try again")
else:
form.add_error(
"__all__",
"There were errors processing your request. "
"Please see below for details.",
)
else:
form = ProjectCreateForm(**form_args)
allocation_form = AllocationCreateForm(
initial={"publication_up_to_date": True})
allocation_form.fields[
"publication_up_to_date"].widget = forms.HiddenInput()
funding_formset = FundingFormset(initial=[{}])
consent_form = ConsentForm()
return render(
request,
"projects/create_project.html",
{
"form": form,
"allocation_form": allocation_form,
"funding_formset": funding_formset,
"consent_form": consent_form,
},
)
def is_membership_manager(project, username):
keycloak_client = KeycloakClient()
return get_user_permissions(keycloak_client, username, project)[0]