def scrape_iso_table(db_session: Session, link: str, destination_folder: str, file_name: str, download: bool = True):
html_file_path = os.path.join(destination_folder, file_name)
# get_aws_html gets the HTML from AWS docs and stores it locally, then returns the path
if download:
if os.path.exists(html_file_path):
os.remove(html_file_path)
get_aws_html(link, html_file_path)
raw_scraping_data = RawScrapingData()
with open(html_file_path, "r") as f:
soup = BeautifulSoup(f.read(), "html.parser")
table = soup.find("tbody")
rows = []
for row in table.contents:
if isinstance(row, Tag):
rows.append(row)
for row in rows:
if isinstance(row, Tag):
service_name = clean_service_name(str(row.contents[1].text))
sdk = clean_service_name(str(row.contents[3].text))
if sdk == "Namespaces*" or service_name == "AWS Services":
continue
raw_scraping_data.add_entry_to_database(
db_session=db_session,
compliance_standard_name="ISO",
sdk=sdk,
service_name=clean_service_name(service_name),
)
def scrape_hipaa_table(db_session: Session,
link: str,
destination_folder: str,
file_name: str,
download: bool = True):
html_file_path = os.path.join(destination_folder, file_name)
if os.path.exists(html_file_path):
os.remove(html_file_path)
if download:
if os.path.exists(html_file_path):
os.remove(html_file_path)
get_aws_html(link, html_file_path)
raw_scraping_data = RawScrapingData()
# These show up as list items but are not relevant at all
false_positives = [
"AWS Cloud Security",
"AWS Management Console"
"AWS CloudEndure"
"Amazon CloudWatch SDK Metrics"
"AWS Managed Services",
"AWS Solutions Portfolio",
"AWS Partner Network",
"AWS Careers",
"AWS Support Overview",
]
service_names = []
with open(html_file_path, "r") as f:
soup = BeautifulSoup(f.read(), "html.parser")
for tag in soup.find_all("li"):
cleaned = clean_service_name(tag.text)
if (cleaned.startswith("Amazon") or cleaned.startswith("AWS")
or cleaned.startswith("Elastic")
or cleaned.startswith("Alexa")):
if cleaned not in false_positives:
service_names.append(cleaned)
for service_name in service_names:
raw_scraping_data.add_entry_to_database(
db_session=db_session,
compliance_standard_name="HIPAA",
sdk=
"", # The HIPAA table does not list SDKs. We will update it to match in a second.
service_name=clean_service_name(service_name),
)
def get_service_name(some_cells):
service_name_cell = some_cells[0].contents[1]
if isinstance(service_name_cell, NavigableString):
service_name = str(service_name_cell)
elif isinstance(service_name_cell, Tag):
service_name = service_name_cell.text
else:
service_name = str(service_name_cell)
service_name = clean_service_name(service_name)
return service_name
def test_clean_service_name_tabs_and_newlines(self):
# Make sure tabs and newlines are removed properly
result = clean_service_name('\n\n\t\tAmazon API Gateway\t\n')
self.assertEqual(result, "Amazon API Gateway")
result = clean_service_name('Amazon API Gateway\n')
self.assertTrue(result == "Amazon API Gateway")
def test_clean_service_name_non_breaking_spaces(self):
result = clean_service_name('AWS Amplify\u00a0')
self.assertEqual(result, "AWS Amplify")