def test_transform_html_xml(self):
html_text = """<xml>
<head>
<title>This is title</title>
</head>
<Body>
<h3>License title</h3>
<p>License text</p>
</Body>
</xml>"""
result = """License title
License text"""
self.assertEqual(transform_html(html_text), result)
def test_transform_html_complex(self):
html_text = """<html>
<head>
<style>
* {
background-color: red;
}
</style>
<title>This is title</title>
</head>
<Body>
<h3>License title</h3>
<p>License text</p>
</Body>
</html>"""
result = """License title
License text"""
self.assertEqual(transform_html(html_text), result)
async def fetch_concurrent():
loop = asyncio.get_event_loop()
async with aiohttp.ClientSession() as session:
tasks = []
for dep_id, urls in dep_data.items():
for index, url in enumerate(urls):
tasks.append(
loop.create_task(
fetch(session, url, f"{dep_id}@{index}")))
for result in asyncio.as_completed(tasks):
try:
response = await result
if response:
output, dep_id = response
with open(os.path.join(dir_path, dep_id), "w") as file:
file.write(transform_html(output))
except aiohttp.client_exceptions.ClientConnectorCertificateError:
continue
def test_transform_html_simple(self):
normal_string = "This is normal string"
self.assertEqual(transform_html(f"<p>{normal_string}</p>"),
normal_string)
def test_transform_html_string(self):
normal_string = "This is normal string"
self.assertEqual(transform_html(normal_string), normal_string)