def setup_tls_certs() -> None:
"""Set up full TLS chain for Compass.
Compass currently (as of 14/11/20) doesn't pass the Intermediate certificate it uses.
This is at time of writing the 'Thawte RSA CA 2018', which is in turned signed by DigiCert Global Root CA.
This function includes the Thawte CA cert in the Certifi chain to allow certificate verification to pass.
Yes, it's horrid. TSA plz fix.
"""
thawte_ca_cert_url = "https://thawte.tbs-certificats.com/Thawte_RSA_CA_2018.crt"
certifi_path = Path(certifi.where())
certifi_contents = certifi_path.read_text("UTF-8")
# Check for contents of Thawte CA, if not add
if "Thawte RSA CA 2018" not in certifi_contents:
logger.info(
"Intermediate Certificate for Compass not found - Installing")
# Fetch Thawte CA from known URL, rather than including PEM
ca_request = requests.get(thawte_ca_cert_url, allow_redirects=False)
# Write to certifi PEM
try:
with certifi_path.open("a", encoding="utf-8") as f:
f.write('\n# Label: "Thawte RSA CA 2018"\n')
f.write(ca_request.text)
except IOError as e:
logger.error(
f"Unable to write to certifi PEM: {e.errno} - {e.strerror}")
def report_keep_alive(self, report_page: str):
logger.info(f"Extending Report Session {datetime.datetime.now()}")
keep_alive = re.search(
r'"KeepAliveUrl":"(.*?)"',
report_page).group(1).encode().decode("unicode-escape")
response = self._post(f"{Settings.base_url}{keep_alive}") # NoQA: F841
return keep_alive # response
def get_report_page(self, run_report_url: str) -> bytes:
# TODO what breaks if we don't update user-agent?
# Compass does user-agent sniffing in reports!!!
self._update_headers(
{"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
# Get initial reports page, for export URL and config.
logger.info("Generating report")
report_page = self._get(f"{Settings.base_url}/{run_report_url}")
return report_page.content
async def terminate(self) -> None:
logger.info("Shutting down Redis plugin")
if not self.redis:
return
# gracefully close connection
logger.debug("Closing Redis connection")
self.redis.close()
await self.redis.wait_closed()
logger.debug("Closed Redis connection")
# remove class attributes
del self.redis
def download_report_normal(self, url: str, params: dict,
filename: str) -> bytes:
start = time.time()
csv_export = self._get(url, params=params)
logger.debug(f"Exporting took {time.time() - start}s")
logger.info("Saving report")
try:
Path(filename).write_bytes(csv_export.content) # TODO Debug check
except IOError as e:
logger.error(
f"Unable to write report export: {e.errno} - {e.strerror}")
logger.info("Report Saved")
logger.debug(len(csv_export.content))
return csv_export.content
def change_role(self, new_role: str) -> None:
"""Update role information.
If the user has multiple roles with the same role title, the first is used.
"""
logger.info("Changing role")
# Change role to the specified role number
member_role_number = next(num for num, name in self.roles_dict.items() if name == new_role.strip())
response = self._post(f"{Settings.base_url}/API/ChangeRole", json={"MRN": member_role_number}) # b"false"
logger.debug(f"Compass ChangeRole call returned: {response.json()}")
# Confirm Compass is reporting the changed role number, update auth headers
self._verify_success_update_properties(check_role_number=member_role_number)
logger.info(f"Role updated successfully! Role is now {self.current_role}.")
async def authenticate_user(
username: str, password: str, role: Optional[str],
location: Optional[str]) -> tuple[User, ci.CompassInterface]:
logger.info(f"Logging in to Compass -- {username}")
api = ci.login(username, password, role=role, location=location)
logger.info(f"Successfully authenticated -- {username}")
user = User(
selected_role=api.user.current_role,
logon_info=(username, password, role, location),
asp_net_id=api.user._asp_net_id, # pylint: disable=protected-access
session_id=api.user._session_id, # pylint: disable=protected-access
props=api.user.compass_props.master.user,
expires=int(time.time() +
9.5 * 60), # Compass timeout is 10m, use 9.5 here
)
return user, api
def _logon_remote(self, auth: tuple[str, str]) -> requests.Response:
"""Log in to Compass and confirm success."""
# Referer is genuinely needed otherwise login doesn't work
headers = {"Referer": f"{Settings.base_url}/login/User/Login"}
username, password = auth
credentials = {
"EM": f"{username}", # assume email?
"PW": f"{password}", # password
"ON": f"{Settings.org_number}", # organisation number
}
# log in
logger.info("Logging in")
response = self._post(f"{Settings.base_url}/Login.ashx", headers=headers, data=credentials)
# verify log in was successful
self._verify_success_update_properties()
return response
async def setup_redis(
self, app: FastAPI, config: RedisSettings = RedisSettings()) -> None:
logger.info("Setting up Redis plugin")
if config.type != "redis":
raise NotImplementedError(
f"Invalid Redis type '{config.type}' selected!")
logger.debug(f"Creating connection to Redis at {config.url}")
self.redis = await create_redis_pool(
config.url.lower(),
db=config.db,
password=config.password,
minsize=config.pool_min_size,
maxsize=config.pool_max_size,
timeout=config.connection_timeout,
ssl=config.ssl,
)
logger.debug("Storing redis object in FastAPI app state")
app.state.redis = self.redis
def get_report(self, report_type: TYPES_REPORTS) -> bytes:
"""Exports report as CSV from Compass.
Exporting a report is of course surprisingly complicated. The process
has four major steps, as follows:
1. Get a token for generating the report from the Compass backend. This
also validates that the report exists and that the user is
authenticated to access it.
2. If successful in obtaining a report token, get the initial report
page. The token is the (relative) URL here, and the report page
contains further needed information, such as the export URL and
location data for a full export.
(Compass does not include all organisational units in reports by
default, and to export all data for a given unit and downwards, we
need to add in these missing/unset levels manually).
3. We update report configuration data (sent as form data), and check
that we are not in an error state.
4. We extract the export URL, download the content and save to disk.
Pitfalls to be aware of in this process include that:
- Compass checks user-agent headers in some parts of the process
(TODO pinpoint which exactly)
- There is a ten (10) minute default soft-timeout, which may run out
before a report download has finished
- If a requested report is too large, Compass can simply give up, often
with an `OutOfMemory` error or similar
Returns:
Report output, as a bytes-encoded object.
Raises:
CompassReportError:
- If the user passes an invalid report type
- If Compass returns a JSON error
- If there is an error updating the form data
CompassReportPermissionError:
If the user does not have permission to run the report
requests.HTTPError:
If there is an error in the transport layer, or if Compass
reports a HTTP 5XX status code
"""
# Get token for report type & role running said report:
try:
# report_type is given as `Title Case` with spaces, enum keys are in `snake_case`
rt_key = report_type.lower().replace(" ", "_")
run_report_url = self._scraper.get_report_token(
ReportTypes[rt_key].value, self.session.mrn)
except KeyError:
# enum keys are in `snake_case`, output types as `Title Case` with spaces
types = [rt.name.title().replace("_", " ") for rt in ReportTypes]
raise CompassReportError(
f"{report_type} is not a valid report type. Existing report types are {types}"
) from None
# Get initial reports page, for export URL and config:
report_page = self._scraper.get_report_page(run_report_url)
# Update form data & set location selection:
self._scraper.update_form_data(
report_page, f"{Settings.base_url}/{run_report_url}")
# Export the report:
logger.info("Exporting report")
export_url_path, export_url_params = self._scraper.get_report_export_url(
report_page.decode("UTF-8"))
# TODO Debug check
time_string = datetime.datetime.now().replace(
microsecond=0).isoformat().replace(
":", "-") # colons are illegal on windows
filename = f"{time_string} - {self.session.cn} ({self.session.current_role}).csv"
# start = time.time()
# TODO TRAINING REPORT ETC.
# # TODO REPORT BODY HAS KEEP ALIVE URL KeepAliveUrl
# p = PeriodicTimer(15, lambda: self.report_keep_alive(self.session, report_page.text))
# self.session.sto_thread.start()
# p.start()
# # ska_url = self.report_keep_alive(self.session, report_page.text)
# try:
# self.download_report(self.session, f"{Settings.base_url}/{export_url_path}", export_url_params, filename, ) # ska_url
# except (ConnectionResetError, requests.ConnectionError):
# logger.info(f"Stopped at {datetime.datetime.now()}")
# p.cancel()
# self.session.sto_thread.cancel()
# raise
# logger.debug(f"Exporting took {time.time() -start}s")
csv_export = self._scraper.download_report_normal(
f"{Settings.base_url}/{export_url_path}", export_url_params,
filename)
return csv_export