def get_members_in_units( self, parent_id: int, compass_ids: Iterable ) -> list[Union[gamih_pydantic, gamih_native]]: with contextlib.suppress(FileNotFoundError): # Attempt to see if the members dict has been fetched already and is on the local system with open(f"all-members-{parent_id}.json", "r", encoding="utf-8") as f: all_members = json.load(f) if all_members: return all_members # Fetch all members all_members = [] for compass_id in set(compass_ids): logger.debug(f"Getting members for {compass_id}") all_members.append( dict(compass_id=compass_id, member=self._scraper.get_members_with_roles_in_unit( compass_id))) # Try and write to a file for caching try: with open(f"all-members-{parent_id}.json", "w", encoding="utf-8") as f: json.dump(all_members, f, ensure_ascii=False, indent=4) except IOError as e: logger.error( f"Unable to write cache file: {e.errno} - {e.strerror}") if self.validate: return schema.HierarchyUnitMembersList.parse_obj( all_members).__root__ else: return all_members
async def get_member_roles(compass_id: int, api: ci.CompassInterface = Depends(ci_user)): """Gets roles for the member given by `compass_id`.""" logger.debug( f"Getting /{{compass_id}}/roles for {api.user.membership_number}") async with error_handler: return api.people.roles(compass_id, only_volunteer_roles=False)
async def get_unit_members( unit_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> list[schema.HierarchyMember]: """Gets hierarchy details for given unit ID.""" logger.debug(f"Getting /hierarchy/{{unit_id}} for {unit_id=}") async with error_handler: return api.hierarchy.unit_members(unit_id)
async def get_member( compass_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> member.MemberDetails: """Gets personal details for the member given by `compass_id`.""" logger.debug(f"Getting /{{compass_id}} for {api.user.membership_number}") async with error_handler: return api.people.personal(compass_id)
async def create_token(username: str, pw: str, role: Optional[str], location: Optional[str], store: Redis) -> str: try: user, _ = await authenticate_user(username, pw, role, location) except ci.errors.CompassError: raise auth_error("A10", "Incorrect username or password!") access_token_expire_minutes = 30 jwt_expiry_time = int(time.time()) + access_token_expire_minutes * 60 to_encode = dict(sub=f"{user.props.cn}", exp=jwt_expiry_time) access_token = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) data = await encrypt(user.json().encode()) encoded = base64.b85encode(data) logger.debug( f"Created JWT for user {username}. Redis key={access_token}, data={encoded}" ) logger.debug(f"Writing {username}'s session key to redis!") asyncio.create_task( store_kv(access_token, data, store, expire_seconds=access_token_expire_minutes * 60)) return access_token
async def get_current_member_ongoing_learning( api: ci.CompassInterface = Depends(ci_user) ) -> member.MemberMandatoryTraining: """Gets my ongoing learning.""" logger.debug(f"Getting /me/ongoing for {api.user.membership_number}") async with error_handler: return api.people.ongoing_learning(api.user.membership_number)
async def get_awards( compass_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> list[member.MemberAward]: """Gets awards for the member given by `compass_id`.""" logger.debug( f"Getting /{{compass_id}}/awards for {api.user.membership_number}") async with error_handler: return api.people.awards(compass_id)
async def get_current_member_roles(api: ci.CompassInterface = Depends(ci_user), volunteer_only: bool = False ) -> member.MemberRolesCollection: """Gets my roles.""" logger.debug(f"Getting /me/roles for {api.user.membership_number}") async with error_handler: return api.people.roles(api.user.membership_number, only_volunteer_roles=volunteer_only)
async def get_current_member_latest_disclosure( api: ci.CompassInterface = Depends(ci_user) ) -> Optional[member.MemberDisclosure]: """Gets my latest disclosure.""" logger.debug( f"Getting /me/latest-disclosure for {api.user.membership_number}") async with error_handler: return api.people.latest_disclosure(api.user.membership_number)
async def get_training( compass_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> member.MemberTrainingTab: """Gets training for the member given by `compass_id`.""" logger.debug( f"Getting /{{compass_id}}/training for {api.user.membership_number}") async with error_handler: return api.people.training(compass_id)
async def get_ongoing_learning( compass_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> member.MemberMandatoryTraining: """Gets ongoing learning for the member given by `compass_id`.""" logger.debug( f"Getting /{{compass_id}}/ongoing-learning for {api.user.membership_number}" ) async with error_handler: return api.people.ongoing_learning(compass_id)
async def get_latest_disclosure( compass_id: int, api: ci.CompassInterface = Depends(ci_user) ) -> Optional[member.MemberDisclosure]: """Gets the latest disclosure for the member given by `compass_id`.""" logger.debug( f"Getting /{{compass_id}}/latest-disclosure for {api.user.membership_number}" ) async with error_handler: return api.people.latest_disclosure(compass_id)
def _jk_hash(self) -> str: """Generate JK Hash needed by Compass.""" # hash_code(f"{time.time() * 1000:.0f}") member_no = self.cn key_hash = f"{time.time() * 1000:.0f}{self.jk}{self.mrn}{member_no}" # JK, MRN & CN are all required. data = compass_restify({"pKeyHash": key_hash, "pCN": member_no}) logger.debug(f"Sending preflight data {datetime.datetime.now()}") self._post(f"{Settings.base_url}/System/Preflight", json=data) return key_hash
async def get_current_user(request: requests.Request, token: str) -> ci.CompassInterface: try: payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) except JWTError: raise auth_error("A20", "Could not validate credentials") if not {"sub", "exp"} <= payload.keys(): raise auth_error("A26", "Your token is malformed! Please get a new token.") if time.time() > payload["exp"]: raise auth_error("A26", "Your token has expired! Please get a new token.") logger.debug(f"Getting data from token:{token}") try: # try fast-path session_decoded = SESSION_STORE.joinpath(f"{token}.bin").read_bytes() except (FileNotFoundError, IOError): store = request.app.state.redis session_encoded = await store.get(f"session:{token}") try: session_decoded = base64.b85decode(session_encoded) except ValueError: raise auth_error("A21", "Could not validate credentials") try: session_decrypted = aes_gcm.decrypt(session_decoded[:12], session_decoded[12:], None) except InvalidTag: raise auth_error("A22", "Could not validate credentials") try: user = User.parse_raw(session_decrypted) logger.debug(f"Created parsed user object {user.__dict__}") except KeyError: raise auth_error("A23", "Could not validate credentials") if time.time() < user.expires: api = ci.CompassInterface( ci.Logon.from_session(user.asp_net_id, user.props.__dict__, user.session_id, user.selected_role)) else: user, api = await authenticate_user(*user.logon_info) asyncio.create_task( store_kv(token, await encrypt(user.json().encode()))) try: if int(payload["sub"]) == int(api.user.membership_number): return api raise auth_error( "A24", "Could not validate credentials") # this should be impossible except ValueError: raise auth_error("A25", "Could not validate credentials")
async def terminate(self) -> None: logger.info("Shutting down Redis plugin") if not self.redis: return # gracefully close connection logger.debug("Closing Redis connection") self.redis.close() await self.redis.wait_closed() logger.debug("Closed Redis connection") # remove class attributes del self.redis
def change_role(self, new_role: str) -> None: """Update role information. If the user has multiple roles with the same role title, the first is used. """ logger.info("Changing role") # Change role to the specified role number member_role_number = next(num for num, name in self.roles_dict.items() if name == new_role.strip()) response = self._post(f"{Settings.base_url}/API/ChangeRole", json={"MRN": member_role_number}) # b"false" logger.debug(f"Compass ChangeRole call returned: {response.json()}") # Confirm Compass is reporting the changed role number, update auth headers self._verify_success_update_properties(check_role_number=member_role_number) logger.info(f"Role updated successfully! Role is now {self.current_role}.")
async def login_for_access_token( request: requests.Request, form_data: auth.OAuth2Details = auth.Form(...) ) -> auth.Token: logger.debug( f"Requested token endpoint with form data: {form_data.__dict__}") try: store = request.app.state.redis access_token = await create_token(form_data.username, form_data.password, form_data.role, form_data.location, store) except exceptions.HTTPException as err: raise err except Exception: raise http_error(status.HTTP_500_INTERNAL_SERVER_ERROR, "A1", "Authentication error!") from None return auth.Token(access_token=access_token)
def download_report_normal(self, url: str, params: dict, filename: str) -> bytes: start = time.time() csv_export = self._get(url, params=params) logger.debug(f"Exporting took {time.time() - start}s") logger.info("Saving report") try: Path(filename).write_bytes(csv_export.content) # TODO Debug check except IOError as e: logger.error( f"Unable to write report export: {e.errno} - {e.strerror}") logger.info("Report Saved") logger.debug(len(csv_export.content)) return csv_export.content
def _get_descendants_recursive( self, compass_id: int, hier_level: Optional[TYPES_UNIT_LEVELS] = None, hier_num: Optional[Levels] = None ) -> dict[str, Union[int, str, None]]: """Recursively get all children from given unit ID and level name/number, with caching.""" if hier_level is hier_num is None: raise ValueError( "A numeric or string hierarchy level needs to be passed") try: level_numeric = hier_num or Levels[ hier_level] # If hier_num is None, hier_level will be used except KeyError: raise ValueError( f"Passed level: {hier_level} is illegal. Valid values are {[level.name for level in Levels]}" ) logger.debug(f"getting data for unit {compass_id}") descendants = level_numeric in set( UnitChildren ) # Do child units exist? (i.e. is this level != group) # All to handle as Group doesn't have grand-children descendant_data = { "id": compass_id, "level": level_numeric.name, "child": self._scraper.get_units_from_hierarchy( compass_id, UnitChildren(level_numeric).name) if descendants else None, "sections": self._scraper.get_units_from_hierarchy( compass_id, UnitSections(level_numeric).name), } child_level = Levels(level_numeric + 1) if descendants else None for child in descendant_data.get("child") or []: grandchildren = self._get_descendants_recursive( child["id"], hier_num=child_level) child.update(grandchildren) return descendant_data
def get_unit_data( self, unit_level: Optional[schema.HierarchyLevel] = None, _id: Optional[int] = None, level: Optional[str] = None, use_default: bool = False, ) -> schema.HierarchyLevel: """Helper function to construct unit level data. Unit data can be specified as a pre-constructed model, by passing literals, or by signalling to use the data from the user's current role. If all three options are unset an exception is raised. There is a strict priority ordering as follows: 1. pre-constructed pydantic model 2. literals 3. default data Returns: Constructed unit level data, as a pydantic model. e.g.: HierarchyLevel(id=..., level="...") Raises: ValueError: When no unit data information has been provided """ if unit_level is not None: data = unit_level elif id is not None and level is not None: data = schema.HierarchyLevel(id=_id, level=level) elif use_default: data = self.session.hierarchy # as this is a property, it will update when roles change else: raise ValueError( "No level data specified! unit_level, id and level, or use_default must be set!" ) logger.debug(f"found unit data: id: {data.id}, level: {data.level}") return data
async def setup_redis( self, app: FastAPI, config: RedisSettings = RedisSettings()) -> None: logger.info("Setting up Redis plugin") if config.type != "redis": raise NotImplementedError( f"Invalid Redis type '{config.type}' selected!") logger.debug(f"Creating connection to Redis at {config.url}") self.redis = await create_redis_pool( config.url.lower(), db=config.db, password=config.password, minsize=config.pool_min_size, maxsize=config.pool_max_size, timeout=config.connection_timeout, ssl=config.ssl, ) logger.debug("Storing redis object in FastAPI app state") app.state.redis = self.redis
def get_report_token(self, report_number: int, role_number: int) -> str: params = { "pReportNumber": report_number, "pMemberRoleNumber": role_number, } logger.debug("Getting report token") response = self._get(f"{Settings.web_service_path}/ReportToken", auth_header=True, params=params) response.raise_for_status() report_token_uri = response.json().get("d") if report_token_uri not in {"-1", "-2", "-3", "-4"}: return report_token_uri elif report_token_uri in {"-2", "-3"}: raise CompassReportError( "Report aborted: Report No Longer Available") elif report_token_uri == "-4": raise CompassReportPermissionError( "Report aborted: USER DOES NOT HAVE PERMISSION") raise CompassReportError("Report aborted")
def _verify_success_update_properties(self, check_role_number: int = None) -> None: """Confirms success and updates authorisation.""" # Test 'get' for an exemplar page that needs authorisation. portal_url = f"{Settings.base_url}/MemberProfile.aspx?Page=ROLES&TAB" response = self._get(portal_url) # # Response body is login page for failure (~8Kb), but success is a 300 byte page. # if int(post_response.headers.get("content-length", 901)) > 900: # raise CompassAuthenticationError("Login has failed") # Naive check for error, Compass redirects to an error page when something goes wrong # TODO what is the error page URL - what do we expect? From memory Error.aspx if response.url != portal_url: raise CompassAuthenticationError("Login has failed") # Create lxml html.FormElement form = html.fromstring(response.content).forms[0] # Update session dicts with new role self.compass_dict = self._create_compass_dict(form) # Updates MRN property etc. self.roles_dict = self._create_roles_dict(form) # Set auth headers for new role auth_headers = { "Authorization": f"{self.cn}~{self.mrn}", "SID": self.compass_dict["Master.Sys.SessionID"], # Session ID } self._update_headers(auth_headers) # Update current role properties self.current_role = self.roles_dict[self.mrn] location = next(row[2].text_content() for row in form.xpath("//tbody/tr") if int(row.get("data-pk")) == self.mrn) logger.debug(f"Using Role: {self.current_role} ({location.strip()})") # Verify role number against test value if check_role_number is not None: logger.debug("Confirming role has been changed") # Check that the role has been changed to the desired role. If not, raise exception. if check_role_number != self.mrn: raise CompassAuthenticationError("Role failed to update in Compass")
async def lifetime(app: FastAPI) -> AsyncGenerator: logger.debug("Initialising RedisPlugin") redis_plugin = RedisPlugin() logger.debug("FastAPI startup: Redis setup") await redis_plugin.setup_redis(app, config=RedisSettings()) yield logger.debug("FastAPI shutdown: Redis teardown") await redis_plugin.terminate()
async def get_current_member_disclosures(api: ci.CompassInterface = Depends( ci_user)) -> list[member.MemberDisclosure]: """Gets my disclosures.""" logger.debug(f"Getting /me/disclosures for {api.user.membership_number}") async with error_handler: return api.people.disclosures(api.user.membership_number)
async def get_current_member_awards(api: ci.CompassInterface = Depends( ci_user)) -> list[member.MemberAward]: """Gets my awards.""" logger.debug(f"Getting /me/awards for {api.user.membership_number}") async with error_handler: return api.people.awards(api.user.membership_number)
def get_roles_tab(self, membership_num: int, keep_non_volunteer_roles: bool = False) -> Union[schema.MemberRolesDict, dict]: """Returns data from Roles tab for a given member. Sanitises the data to a common format, and removes Occasional Helper, Network, and PVG roles by default. Args: membership_num: Membership Number to use keep_non_volunteer_roles: Keep Helper (OH/PVG) & Network roles? Returns: A dict of dicts mapping keys to the corresponding data from the roles tab. E.g.: {1234578: {'role_number': 1234578, 'membership_number': ..., 'role_title': '...', 'role_class': '...', 'role_type': '...', 'location_id': ..., 'location_name': '...', 'role_start_date': datetime.datetime(...), 'role_end': datetime.datetime(...), 'role_status': '...'}, {...} } Keys will always be present. Raises: PermissionError: Access to the member is not given by the current authentication Todo: Other possible exceptions? i.e. from Requests primary_role """ logger.debug(f"getting roles tab for member number: {membership_num}") response = self._get_member_profile_tab(membership_num, "Roles") tree = html.fromstring(response) if tree.forms[0].action == "./ScoutsPortal.aspx?Invalid=AccessCN": raise PermissionError(f"You do not have permission to the details of {membership_num}") roles_data = {} rows = tree.xpath("//tbody/tr") for row in rows: # Get children (cells in row) cells = list(row) # filter out empty elements # If current role allows selection of role for editing, remove tickbox if any(el.tag == "input" for el in cells[0]): cells.pop(0) role_number = int(row.get("data-pk")) status_with_review = cells[5].text_content().strip() if status_with_review.startswith("Full Review Due "): role_status = "Full" review_date = parse(status_with_review.removeprefix("Full Review Due ")) else: role_status = status_with_review review_date = None role_details = dict( role_number=role_number, membership_number=membership_num, role_title=cells[0].text_content().strip(), role_class=cells[1].text_content().strip(), # role_type only visible if access to System Admin tab role_type=[*row.xpath("./td[1]/*/@title"), None][0], # location_id only visible if role is in hierarchy AND location still exists location_id=cells[2][0].get("data-ng_id"), location_name=cells[2].text_content().strip(), role_start=parse(cells[3].text_content().strip()), role_end=parse(cells[4].text_content().strip()), role_status=role_status, review_date=review_date, can_view_details=any("VIEWROLE" in el.get("class") for el in cells[6]), ) # Remove OHs etc from list if not keep_non_volunteer_roles and ( "helper" in role_details["role_class"].lower() or {role_details["role_title"].lower()} <= {"occasional helper", "pvg", "network member"} ): continue roles_data[role_number] = role_details if self.validate: return schema.MemberRolesDict.parse_obj(roles_data) else: return roles_data
async def get_current_member_training(api: ci.CompassInterface = Depends( ci_user)) -> member.MemberTrainingTab: """Gets my training.""" logger.debug(f"Getting /me/training for {api.user.membership_number}") async with error_handler: return api.people.training(api.user.membership_number)
def get_roles_detail( self, role_number: int, response: Union[str, requests.Response] = None ) -> Union[schema.MemberRolePopup, dict]: """Returns detailed data from a given role number. Args: role_number: Role Number to use response: Pre-generated response to use Returns: A dicts mapping keys to the corresponding data from the role detail data. E.g.: {'hierarchy': {'organisation': 'The Scout Association', 'country': '...', 'region': '...', 'county': '...', 'district': '...', 'group': '...', 'section': '...'}, 'details': {'role_number': ..., 'organisation_level': '...', 'birth_date': datetime.datetime(...), 'membership_number': ..., 'name': '...', 'role_title': '...', 'role_start': datetime.datetime(...), 'role_status': '...', 'line_manager_number': ..., 'line_manager': '...', 'ce_check': datetime.datetime(...), 'disclosure_check': '...', 'references': '...', 'appointment_panel_approval': '...', 'commissioner_approval': '...', 'committee_approval': '...'}, 'getting_started': {...: {'name': '...', 'validated': datetime.datetime(...), 'validated_by': '...'}, ... }} Keys will always be present. Todo: Other possible exceptions? i.e. from Requests """ # pylint: disable=too-many-locals,too-many-statements renamed_levels = { "County / Area / Scottish Region / Overseas Branch": "County", } renamed_modules = { 1: "module_01", "TRST": "trustee_intro", 2: "module_02", 3: "module_03", 4: "module_04", "GDPR": "GDPR", } unset_vals = {"--- Not Selected ---", "--- No Items Available ---", "--- No Line Manager ---"} module_names = { "Essential Information": "M01", "Trustee Introduction": "TRST", "PersonalLearningPlan": "M02", "Tools for the Role (Section Leaders)": "M03", "Tools for the Role (Managers and Supporters)": "M04", "General Data Protection Regulations": "GDPR", } references_codes = { "NC": "Not Complete", "NR": "Not Required", "RR": "References Requested", "S": "References Satisfactory", "U": "References Unsatisfactory", } start_time = time.time() if response is None: response = self._get(f"{Settings.base_url}/Popups/Profile/AssignNewRole.aspx?VIEW={role_number}") logger.debug(f"Getting details for role number: {role_number}. Request in {(time.time() - start_time):.2f}s") post_response_time = time.time() if isinstance(response, (str, bytes)): tree = html.fromstring(response) else: tree = html.fromstring(response.content) form = tree.forms[0] if form.action == "./ScoutsPortal.aspx?Invalid=Access": raise PermissionError(f"You do not have permission to the details of role {role_number}") member_string = form.fields.get("ctl00$workarea$txt_p1_membername") ref_code = form.fields.get("ctl00$workarea$cbo_p2_referee_status") role_details = dict() # Approval and Role details role_details["role_number"] = role_number role_details["organisation_level"] = form.fields.get("ctl00$workarea$cbo_p1_level") role_details["birth_date"] = parse(form.inputs["ctl00$workarea$txt_p1_membername"].get("data-dob")) role_details["membership_number"] = int(form.fields.get("ctl00$workarea$txt_p1_memberno")) role_details["name"] = member_string.split(" ", maxsplit=1)[1] # TODO does this make sense - should name be in every role?? role_details["role_title"] = form.fields.get("ctl00$workarea$txt_p1_alt_title") role_details["role_start"] = parse(form.fields.get("ctl00$workarea$txt_p1_startdate")) # Role Status role_details["role_status"] = form.fields.get("ctl00$workarea$txt_p2_status") # Line Manager line_manager_el = next((op for op in form.inputs["ctl00$workarea$cbo_p2_linemaneger"] if op.get("selected")), None) role_details["line_manager_number"] = maybe_int(line_manager_el.get("value")) if line_manager_el is not None else None role_details["line_manager"] = line_manager_el.text.strip() if line_manager_el is not None else None # Review Date role_details["review_date"] = parse(form.fields.get("ctl00$workarea$txt_p2_review")) # CE (Confidential Enquiry) Check # TODO if CE check date != current date then is valid role_details["ce_check"] = parse(form.fields.get("ctl00$workarea$txt_p2_cecheck")) # Disclosure Check disclosure_with_date = form.fields.get("ctl00$workarea$txt_p2_disclosure") if disclosure_with_date.startswith("Disclosure Issued : "): disclosure_date = parse(disclosure_with_date.removeprefix("Disclosure Issued : ")) disclosure_check = "Disclosure Issued" else: disclosure_date = None disclosure_check = disclosure_with_date role_details["disclosure_check"] = disclosure_check # TODO extract date role_details["disclosure_date"] = disclosure_date # TODO extract date # References role_details["references"] = references_codes.get(ref_code, ref_code) approval_values = {} for row in tree.xpath("//tr[@class='trProp']"): select = row[1][0] code = select.get("data-app_code") approval_values[code] = select.get("data-db") # select.get("title") gives title text, but this is not useful as it does not reflect latest changes, # but only who added the role to Compass. # Appointment Panel Approval role_details["appointment_panel_approval"] = approval_values.get("ROLPRP|AACA") # Commissioner Approval role_details["commissioner_approval"] = approval_values.get("ROLPRP|CAPR") # Committee Approval role_details["committee_approval"] = approval_values.get("ROLPRP|CCA") if role_details["line_manager_number"] in unset_vals: role_details["line_manager_number"] = None # Filter null values role_details = {k: v for k, v in role_details.items() if v is not None} # Getting Started modules_output = {} getting_started_modules = tree.xpath("//tr[@class='trTrain trTrainData']") # Get all training modules and then extract the required modules to a dictionary for module in getting_started_modules: module_name = module[0][0].text.strip() if module_name in module_names: info = { # "name": module_names[module_name], # short_name "validated": parse(module[2][0].value), # Save module validation date "validated_by": module[1][1].value or None, # Save who validated the module } mod_code = cast(module[2][0].get("data-ng_value")) # int or str modules_output[renamed_modules[mod_code]] = info # Get all levels of the org hierarchy and select those that will have information: # Get all inputs with location data org_levels = [v for k, v in sorted(dict(form.inputs).items()) if "ctl00$workarea$cbo_p1_location" in k] # TODO all_locations = {row.get("title"): row.findtext("./option") for row in org_levels} clipped_locations = { renamed_levels.get(key, key).lower(): value for key, value in all_locations.items() if value not in unset_vals } logger.debug( f"Processed details for role number: {role_number}. " f"Compass: {(post_response_time - start_time):.3f}s; Processing: {(time.time() - post_response_time):.4f}s" ) # TODO data-ng_id?, data-rtrn_id? full_details = { "hierarchy": clipped_locations, "details": role_details, "getting_started": modules_output, } if self.validate: return schema.MemberRolePopup.parse_obj(full_details) else: return full_details
async def get_current_member(api: ci.CompassInterface = Depends( ci_user)) -> member.MemberDetails: """Gets my personal details.""" logger.debug(f"Getting /me for {api.user.membership_number}") async with error_handler: return api.people.personal(api.user.membership_number)