async def deserialize(request_result) -> pd.DataFrame:
request_result = await request_result
pd_from_bytes = lambda b: pd.read_parquet(BytesIO(b))
try:
return request_result.either(Left, lambda bytes: Right(pd_from_bytes(bytes)))
except OSError:
return Left(DeserializationError(request_result.either(str,str)))
async def get(session: aiohttp.ClientSession, url: str)->Either:
async with session.get(url) as response:
content = await response.content.read()
if (status := response.status) == 200:
return Right(content)
elif status == 202:
return Left(Pending(url))
def scrape_bill_from_in_prog(
bill_p: BillProgress) -> Tuple[BillProgress, Either[str, Bill]]:
try:
_d: TypedDict[Bill] = AttrDict(name=bill_p.name,
bill_no=bill_p.bill_no,
url=bill_p.url,
bill_progress=Just(bill_p))
page = BeautifulSoup(reqs_get_wa_parli(bill_p.url).text, 'lxml')
# name, bill_no, synopsys, status
heading = page.find('table', {'class': 'billHeading'})
if heading is None:
print(heading)
print(page)
heading_tds = list(heading.find_all('td'))
_d.name = heading_tds[0].text.strip()
_d.bill_no = heading_tds[2].text.strip()
_d.synopsis = heading_tds[4].text.strip()
_d.private_members_bill = Just(
heading_tds[5].text.strip()) if len(heading_tds) != 7 else Nothing
_d.status = heading_tds[6 if _d.private_members_bill.is_nothing(
) else 7].text.strip()
main_info = heading.find_next_sibling('table')
top_level_trs = main_info.find_all(
'tr',
recursive=False,
)
main_row_paris = chunks(top_level_trs, 2)
_d.bill_history = li_extract(main_row_paris.__next__()[1])
_d.acts_amended = li_extract(main_row_paris.__next__()[1])
_d.related_committee_activity = li_extract(
main_row_paris.__next__()[1])
_d.lc_supplementary = li_extract(main_row_paris.__next__()[1])
_d.messages = li_extract(main_row_paris.__next__()[1])
progress_table = main_row_paris.__next__()[1].find('table')
la_table = None if progress_table is None else progress_table.find(
'table', {'class': 'bil_table_LA'})
lc_table = None if progress_table is None else progress_table.find(
'table', {'class': 'bil_table_LC'})
_d.progress_la = progress_extract(la_table)
_d.progress_lc = progress_extract(lc_table)
superseded_pair = main_row_paris.__next__()
_d.superseded_lc_supplementary = li_extract(superseded_pair[1])
comparison_1: Tag = main_row_paris.__next__()[1]
comparison_2: Tag = main_row_paris.__next__()[0]
# we're now offset by 1 to account for an extra row
_d.comparisons_between_version = (li_extract(comparison_1),
li_extract(comparison_2))
_d.conference_of_managers = li_extract(main_row_paris.__next__()[0])
if len(_d.conference_of_managers) > 0:
print(f"found conference_of_managers? {_d.conference_of_managers}")
_d.notes = list(td.text
for td in main_row_paris.__next__()[0].find_all('td'))
return bill_p, Right(Bill(**_d))
except Exception as e:
import traceback
log.error(
f"scrape_bill_from_in_prog ({bill_p.name}) Error: {e}\n\n{traceback.format_tb(e.__traceback__)}"
)
return bill_p, Left(str(e))
async def mySafeComputation(x: int):
if x > 5:
return Left(x)
waiter = x % 2 + 1
print(f"Computation with {x} - {waiter}")
await asyncio.sleep(waiter)
print(f"Results of {x}")
return Right(x + 1)
async def mySafeComputation(x: int):
if np.random.randint(0,10)>8:
return Left(RandomError(f"Random Error with {x}"))
waiter = np.random.randint(0,5) # x % 2 + 1
print(f"Computation with {x} - {waiter}")
await asyncio.sleep(waiter)
print(f"Results of {x}")
return Right(x+1)
async def fetch_set(base_url: str, queryset: models.Queryset)-> Either:
async with aiohttp.ClientSession() as session:
get_data = compose(
deserialize,
curry(get,session),
)
results = await asyncio.gather(
*map(get_data, make_urls(base_url, queryset))
)
unpack_errors = compose(
curry(filter,lambda x: x is not None),
curry(map,lambda e: e.either(identity, lambda _: None))
)
errors = [*unpack_errors(results)]
if len(errors)>0:
return Left(errors)
else:
return Right(fast_views.inner_join([res.value for res in results]))
def test_mapping_over_left(self):
self.assertEqual(Left('').map(lambda x: x), Left(''))
def test_either_extraction_with_Left_value(self):
self.assertEqual(
Left(TypeError()).either(lambda e: 'Left', lambda r: 'Right'),
'Left')
def test_repr(self):
self.assertEqual(str(Right(9)), 'Right 9')
self.assertEqual(str(Left(9)), 'Left 9')
async def bad_request_500(_, __):
return Left(
retrieval.HTTPNotOk("http://foobar", 500, "something went wrong"))
def wrap(*args, **kwargs) -> Either:
try:
return Right(f(*args, **kwargs))
except Exception as e:
return Left(e)
def errorHandling(x):
print(f"Error {x}")
return Left(x)
def test_exceptions_wrapped_in_left_kleisli_function(self):
self.assertEqual(str(Either.insert(1).then(lambda x: Right(x / 0))),
str(Left(ZeroDivisionError('division by zero'))))
def mustBeLowerThan(value: int, x) -> Either:
if (x < value):
return Right(x)
else:
return Left(TooHigh(x))
def test_applying_with_left_in_second_arg(self):
self.assertEqual(
Either.apply(common_tests.add).to_arguments(Right(1), Left('')),
Left(''))
def test_binding_with_left(self):
self.assertEqual(Left('').bind(Either.insert), Left(''))
self.status_code = 500
super().__init__(f"Could not deserialize as parquet: {bytes}")
class Pending(Exception):
def __init__(self,url):
super().__init__(f"{url} is pending")
async def get(session: aiohttp.ClientSession, url: str)->Either:
async with session.get(url) as response:
content = await response.content.read()
if (status := response.status) == 200:
return Right(content)
elif status == 202:
return Left(Pending(url))
else:
return Left(HTTPNotOk(url,status, content))
def make_urls(base_url: str, queryset):
return [base_url + "/" + path for path in queryset.paths()]
async def deserialize(request_result) -> pd.DataFrame:
request_result = await request_result
pd_from_bytes = lambda b: pd.read_parquet(BytesIO(b))
try:
return request_result.either(Left, lambda bytes: Right(pd_from_bytes(bytes)))
except OSError:
return Left(DeserializationError(request_result.either(str,str)))
async def fetch_set(base_url: str, queryset: models.Queryset)-> Either:
async with aiohttp.ClientSession() as session:
get_data = compose(