def response(self):
"""
Submits and gets response for SIFO request
Returns
-------
out : requests.Response
response with expenses information
"""
try:
start = time()
parsed_sifo_url = SIFO_URL
for key, item in self.family.sifo_properties().items():
parsed_sifo_url = parsed_sifo_url + key + '=' + item + '&'
response = requests.post(url=parsed_sifo_url, timeout=TIMEOUT)
status_code = response.status_code
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> SIFO: [{}: {}] -> elapsed: {}".format(
status_code, responses[status_code], elapsed))
return response
except URLError as sifo_response_error:
if str(sifo_response_error) == "<urlopen error timed out>":
raise TimeOutError(
"Timeout occurred - please try again later or contact system "
"administrator, exited with '{}'".format(
sifo_response_error))
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".
format(sifo_response_error))
def response(self):
"""
Submits and gets response for posten request
Returns
-------
out : mechanize._response.response_seek_wrapper
response with expenses information
"""
try:
try:
start = time()
posten_response = requests.get(POSTEN_URL +
"{}".format(self.postal_code),
timeout=TIMEOUT)
posten_status_code = posten_response.status_code
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> POSTEN: [{}: {}] -> elapsed: {}".
format(posten_status_code, responses[posten_status_code],
elapsed))
return posten_response
except ConnectTimeout as posten_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or contact system "
"administrator, exited with '{}'".format(
posten_timeout_error))
except ConnectError as posten_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".
format(posten_response_error))
def sifo_base_expenses(self, include_id: bool = False):
"""
get SIFO base expenses given the family information
Returns
-------
out : dict
dictionary with SIFO expenses
"""
LOGGER.info("trying to retrieve '{}'".format(
self.sifo_base_expenses.__name__))
response_json = self.response().json()["utgifter"]
sifo_expenses = {}
sifo_expenses.update(response_json['individspesifikke'])
sifo_expenses.update({'sumindivid': response_json['sumindivid']})
sifo_expenses.update(response_json['husholdsspesifikke'])
sifo_expenses.update(
{'sumhusholdning': response_json['sumhusholdning']})
sifo_expenses.update({'totalt': response_json['totalt']})
sifo_expenses = {key: str(val) for key, val in sifo_expenses.items()}
if include_id:
sifo_expenses.update({'_id': self.family.id_})
# import json
# with open('sifo_data.json', 'w', encoding='utf-8') as file:
# json.dump(sifo_expenses, file, ensure_ascii=False, indent=4)
LOGGER.success("'{}' successfully retrieved".format(
self.sifo_base_expenses.__name__))
return sifo_expenses
def __init__(self, age: Union[str, int], income: Union[str, int, float]):
"""
Constructor / Instantiate the class
Parameters
----------
age : str, int
age of individual
income : str, int, float
income of individual
"""
try:
super().__init__()
Assertor.assert_data_types([age, income], [(str, int),
(str, int, float)])
self.age = str(age + 1)
self.income = str(income)
self.year = str(2022)
self.url = SKATTEETATEN_URL + self.year
LOGGER.success("created '{}', with id: [{}]".format(
self.__class__.__name__, self.id_))
except Exception as skatteetaten_exception:
LOGGER.exception(skatteetaten_exception)
raise skatteetaten_exception
def postal_code_info(self):
"""
gets postal code information
Returns
-------
out : dict
dictionary with postal code information
"""
LOGGER.info("trying to retrieve 'postal_code_info' for -> '{}'".format(
self.postal_code))
soup = BeautifulSoup(self.response().content, "lxml")
rows = soup.find_all("tr")
# with open('content.html', 'w', encoding='utf-8') as file:
# file.write(soup.prettify())
if len(rows) == 2:
header = [
head.text.strip().lower() for head in soup.find_all('th')
]
values = [
value.text.strip().upper()
if i != 4 else value.text.strip().upper().rsplit(' ', 1)[0]
for i, value in enumerate(rows[1].find_all('td'))
]
LOGGER.success("'postal_code_info' successfully retrieved")
return {
hdr: val
for hdr, val in dict(zip(header, values)).items() if val
}
raise NotFoundError("'{}' is an invalid postal code".format(
self.postal_code))
def response(self):
"""
submits and gets response for skatteetaten request
Returns
-------
out : requests.models.Response
response with interest rate information
"""
try:
try:
response = requests.post(url=self.url,
data=self.payload(),
timeout=TIMEOUT)
status_code = response.status_code
LOGGER.info("HTTP status code -> [{}: {}]".format(
status_code, responses[status_code]))
return response
except ConnectTimeout as ssb_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or contact system administrator, "
"exited with '{}'".format(ssb_timeout_error))
except ConnectError as ssb_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".
format(ssb_response_error))
def mortgage_offers(self):
"""
Retrieve finansportalen.no's boliglån grunndata xml and returns dict for content
Returns
-------
out : dict
content from boliglån grunndata Xxml feed
"""
LOGGER.info("trying to retrieve '{}'".format(
self.mortgage_offers.__name__))
response = self.portalen_response()
if response:
tree = Et.fromstring(response.content.decode(
"windows-1252")).findall(PORTALEN_ENTRY)
offers = {}
count = 0
for entries in tree:
count += 1
offers.update({
count: {
re.sub("{[^>]+}", "", entry.tag): entry.text.strip()
for entry in entries if entry.text
}
})
LOGGER.success("'{}' successfully retrieved".format(
self.mortgage_offers.__name__))
return offers
raise NotFoundError("No 'mortgage_offers' received")
def portalen_response(self):
"""
Response from finansportalen.no xml feed
Returns
-------
our : requests.models.Response
response with mortgage information
"""
try:
try:
response = requests.post(PORTALEN_URL,
auth=PORTALEN_CRED,
timeout=TIMEOUT)
status_code = response.status_code
LOGGER.info("HTTP status code -> [{}: {}]".format(
status_code, responses[status_code]))
return response
except ReadTimeout as portalen_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or contact system administrator, "
"exited with '{}'".format(portalen_timeout_error))
except ConnectError as portalen_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".
format(portalen_response_error))
def ad_response(self):
"""
Response from Finn-no ad housing search
Returns
-------
our : requests.models.Response
response with housing ad information
"""
try:
try:
start = time()
ad_response = requests.get(FINN_AD_URL + "{}".format(self.finn_code),
timeout=TIMEOUT)
ad_status_code = ad_response.status_code
if ad_status_code == 404:
ad_response = requests.get(
FINN_AD_URL.replace('homes', 'newbuildings') + "{}".format(self.finn_code),
timeout=TIMEOUT)
ad_status_code = ad_response.status_code
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> ADVERT: [{}: {}] -> elapsed: {}".format(
ad_status_code, responses[ad_status_code], elapsed))
return ad_response
except ConnectTimeout as finn_ad_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or contact system "
"administrator, exited with '{}'".format(finn_ad_timeout_error))
except ConnectError as finn_ad_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".format(
finn_ad_response_error))
async def stat_response(self):
"""
Response from Finn-no housing statistics search
Returns
-------
our : requests.models.Response
response with housing statistics information
"""
try:
try:
start = time()
async with ClientSession(
timeout=ClientTimeout(TIMEOUT)) as session:
async with session.get(
FINN_STAT_URL +
"{}".format(self.finn_code)) as stat_response:
stat_status_code = stat_response.status
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> STATISTICS: [{}: {}] -> elapsed: {}"
.format(stat_status_code,
responses[stat_status_code], elapsed))
return await stat_response.content.read()
except TError:
raise TimeOutError(
"Timeout occurred - please try again later or contact system administrator"
)
except ClientConnectionError as finn_stat_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".
format(finn_stat_response_error))
def __init__(self,
age: Union[int, float, str] = 0,
kinder_garden: str = '0',
sfo: str = '0',
pregnant: str = '0'):
"""
Constructor / Instantiate the class
Parameters
----------
age : int, float, str
age of person
kinder_garden : str
kids in kinder garden, '1' true or '0' false
sfo : str
After school programme, '1' true or '0' false
pregnant : str
Pregnant female, '1' true or '0' false
"""
try:
super().__init__('k', age, kinder_garden, sfo)
Assertor.assert_data_types([age, kinder_garden, sfo],
[(float, int, str), str, str])
Assertor.assert_arguments({pregnant: ['pregnant', ('0', '1')]})
if Person.sifo_age(age) not in ('19', '50') and pregnant == '1':
raise ValueError("pregnancy at this age is not possible")
self._gravid = pregnant
LOGGER.success("created '{}', with id: [{}]".format(
self.__class__.__name__, self.id_str))
except Exception as female_exception:
LOGGER.exception(female_exception)
raise female_exception
def ownership_response(self):
"""
Response from Finn-no housing ownership history search
Returns
-------
our : requests.models.Response
response with housing ownership information
"""
try:
try:
start = time()
owner_response = requests.get(FINN_OWNER_URL + "{}".format(self.finn_code),
timeout=TIMEOUT)
owner_status_code = owner_response.status_code
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> OWNERSHIP HISTORY: [{}: {}] -> elapsed: {}".format(
owner_status_code, responses[owner_status_code], elapsed))
return owner_response
except ConnectTimeout as finn_owner_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or "
"contact system administrator, exited with '{}'".format(
finn_owner_timeout_error))
except ConnectError as finn_owner_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".format(
finn_owner_response_error))
def __init__(self, family_members: list = None, income: Union[int, float, str] = 0,
cars: Union[int, str] = 0):
"""
Constructor / Instantiate the class
Parameters
----------
family_members : list
list of Person (Male or Female) instances
income : int, float, str
gross yearly income
cars : int, str
number of cars in the family
"""
super().__init__()
try:
self._assert_family_members(family_members)
Assertor.assert_data_types([income, cars], [(int, float, str), (int, str)])
Assertor.assert_non_negative([income, cars])
self._family_members = family_members
self._inntekt = str(income)
self._antall_biler = str(cars)
LOGGER.success(
"created '{}', with id: [{}]".format(self.__class__.__name__, self.id_str))
except Exception as family_exception:
LOGGER.exception(family_exception)
raise family_exception
def format_number(self):
"""
formatting of phone number according to norwegian standard
Returns
-------
out : str
formatted phone number
References
-------
https://begrep.difi.no/Felles/mobiltelefonnummer
"""
try:
prefix, number = "+47", self.remove_prefix(self.number)
valid_number = re.compile("^\\+?[- 0-9]{8,20}$").search(number)
if valid_number:
phone_number = prefix + " " + " ".join(
[number[i:i + 2] for i in range(0, len(number), 2)])
LOGGER.info("format number '{}' to -> '{}'".format(
number, phone_number))
return phone_number
raise NotPossibleError(
"'{}' is an invalid phone number".format(number))
except Exception as format_number_error:
LOGGER.exception(format_number_error)
raise format_number_error
def community_stat_response(self):
"""
Response from Finn-no housing statistics search
Returns
-------
our : requests.models.Response
response with housing statistics information
"""
try:
try:
start = time()
community_stat_response = requests.get(
FINN_COMMUNITY_URL + "{}".format(self.finn_code),
timeout=TIMEOUT)
stat_status_code = community_stat_response.status_code
elapsed = self.elapsed_time(start)
LOGGER.info(
"HTTP status code -> COMMUNITY: [{}: {}] -> elapsed: {}".format(
stat_status_code, responses[stat_status_code], elapsed))
return community_stat_response
except ConnectTimeout as finn_community_stat_timeout_error:
raise TimeOutError(
"Timeout occurred - please try again later or contact system "
"administrator, exited with '{}'".format(finn_community_stat_timeout_error))
except ConnectError as finn_community_stat_response_error:
raise NoConnectionError(
"Failed HTTP request - please insure that internet access is provided to the "
"client or contact system administrator, exited with '{}'".format(
finn_community_stat_response_error))
def __init__(self):
"""
Constructor / Instantiate the class. Only one property, i.e. id given by uuid4
"""
LOGGER.info(
"trying to create '{}'".format(self.__class__.__name__))
self._id_str = str(uuid4())
def start_process(cls):
"""
method for starting logging and profiling of process
"""
LOGGER.info("starting '{}'".format(cls.__name__))
cls.start = time()
cls.profiling = profiling_config()
def __init__(self):
"""
Constructor / Instantiate the class
"""
super().__init__()
self._browser = None
LOGGER.success("created '{}', with id: [{}]".format(
self.__class__.__name__, self.id_))
def to_json(self, file_dir: str = "report/json/finn_information"):
"""
save ownership information to JSON file
"""
self.save_json(self.housing_ownership_information(), file_dir,
file_prefix="HousingOwnershipInfo_")
LOGGER.success(
"'housing_ownership_information' successfully parsed to JSON at '{}'".format(file_dir))
def to_json(self, file_dir: str = "report/json/mortgage_offers"):
"""
save mortgage offers information to JSON file
"""
self.save_json(self.housing_information(),
file_dir,
file_prefix="HousingInfo_")
LOGGER.success("'{}' successfully parsed to JSON at '{}'".format(
self.housing_information().__name__, file_dir))
def to_json(self, file_dir: str = "report/json/finn_information"):
"""
save statistics information to JSON file
"""
Assertor.assert_data_types([file_dir], [str])
self.save_json(self.community_stat_information(), file_dir,
file_prefix="CommunityStatInfo_")
LOGGER.success(
"'community_stat_information' successfully parsed to JSON at '{}'".format(file_dir))
def __init__(self):
"""
Abstract class, so class cannot be instantiated
"""
LOGGER.info("trying to create '{}'".format(self.__class__.__name__))
super().__init__()
self._browser = Browser()
self._browser.set_handle_robots(False)
self._browser.set_handle_refresh(False)
self._id = str(uuid4())
def to_json(self, file_dir: str = "report/json/mortgage_offers"):
"""
save mortgage offers information to JSON file
"""
self.save_json(self.mortgage_offers(),
file_dir,
file_prefix="MortgageOffers_")
LOGGER.success(
"'mortgage_offers' successfully parsed to JSON at '{}'".format(
file_dir))
def __init__(self):
"""
Constructor / Instantiate the class
"""
try:
super().__init__()
self._browser = requests.post(PORTALEN_URL, auth=PORTALEN_CRED)
LOGGER.success("created '{}', with id: [{}]".format(
self.__class__.__name__, self.id_str))
except Exception as portalen_exception:
LOGGER.exception(portalen_exception)
raise portalen_exception
def response(self):
"""
Response from finansportalen.no xml feed
Returns
-------
our : requests.models.Response
response with mortgage information
"""
response = self._browser
LOGGER.info("HTTP status code -> [{}: {}]".format(
response.status_code, response.reason))
return response
def response(self):
"""
Response from Finn-no housing search
Returns
-------
our : requests.models.Response
response with mortgage information
"""
response = self._browser
LOGGER.info("HTTP status code -> [{}: {}]".format(
response.status_code, response.reason))
return response
def to_json(self, file_dir: str = "report/json/finn_information"):
"""
save
advert
information
to
JSON
file
"""
Assertor.assert_data_types([file_dir], [str])
self.save_json(self.housing_ad_information(), file_dir, file_prefix="HousingAdInfo_")
LOGGER.success(
"'housing_ad_information' successfully parsed to JSON at '{}'".format(file_dir))
def to_json(self, file_dir: str = "report/json/expenses"):
"""
save sifo expenses to JSON
Parameters
----------
file_dir : str
file directory to save JSON files
"""
self.save_json(self.sifo_expenses(),
file_dir=file_dir,
file_prefix="SifoExpenses_")
LOGGER.success("'{}' successfully parsed to JSON at '{}'".format(
self.sifo_expenses.__name__, file_dir))
def response(self):
"""
submits and gets response for SSB request
Returns
-------
out : requests.models.Response
response with interest rate information
"""
payload = self._payload.payload()
response = requests.post(url=SSB_URL, json=payload)
LOGGER.info("HTTP status code -> [{}: {}]".format(
response.status_code, response.reason))
return response
def to_json(self, file_dir: str = "report/json/interest_rates"):
"""
save ssb interest rate information to JSON
Parameters
----------
file_dir : str
file directory to save JSON files
"""
self.save_json(self.ssb_interest_rates(),
file_dir=file_dir,
file_prefix="SsbInterestRates_")
LOGGER.success("'{}' successfully parsed to JSON at '{}'".format(
self.ssb_interest_rates.__name__, file_dir))