def get_offered_apartments_by_type(offer_id, type):
""" Function to get a list of apartment ids having a given type for the given offer.
Args:
offer_id: id of desired apartment offering.
type: apartment type to use for filtering results.
Returns:
list: list containing the ids of the desired apartments.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
apt_list = db_connection.get_offered_apartments_by_type(offer_id, type)
return apt_list
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
def get_applicants_hist(offer_id, apartment_list):
"""Function to get the time series of the number of applicants for the specified offer and apartments.
Args:
offer_id: id of desired apartment offering.
apartment_list: list of apartment ids whose info we want to retrieve.
Returns:
pandas.DataFrame: data frame containing the "no_applicants" times series for each apartment.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
df = db_connection.get_all_no_applicants(offer_id, apartment_list)
return df
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
def get_offered_apartments(offer_id):
""" Function to get a list of apartment ids for a given offer.
Args:
offer_id: id of desired apartment offering.
Returns:
list: list containing the ids of the desired apartments.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
apt_list = db_connection.get_offered_apartments(offer_id)
return apt_list
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
def spider_closed(self, spider):
"""Method that sets action to do when the spider is closed,
in this case, close database connection and send slack
notification.
Args:
spider: spider calling this pipeline.
"""
db_connection.disconnect()
def scrape_offering(self):
"""Method to systematically obtain the name and offer deadline for each available apartment and
insert it into the database.
"""
if self.logged_in:
no_apts = self.get_no_apartments()
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
# For each apartment
i = 1
# For avoiding dangerous loops
j = 0
while i <= no_apts:
if j >= 5:
raise ApartmentException(
"Cannot get past apartment \"{0}\"".format(
apt_name))
info = self.get_apartment_and_offer(i)
if info is not None:
apt_name = info[0]
end_date_and_time = info[1]
try:
db_connection.set_is_offered(
apt_name, end_date_and_time)
# Only advance to next apartment if the current one was successfully scraped.
i = i + 1
j = 0
except DatabaseException as e:
j = j + 1
print("Failure to insert some data: " + str(e))
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
else:
# Apartments from current offering
print("Cannot get offering. Not logged in.")
def get_last_offer_id():
"""Function to get the id of the current offer.
Returns:
int = id of the desired offer.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
offer_id = db_connection.get_current_offer_id()
return offer_id
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
def get_last_offer_timestamps():
"""Function to get the start and end timestamps of the current offer.
Returns:
datetime.datetime, datetime.datetime = start and ending timestamps respectively.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
start_date, end_date = db_connection.get_current_offer_dates()
return start_date, end_date
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()
def get_db_offering_size():
""" Function to get the number of apartments currently stored in the database
fr the last available offering.
Returns:
int: number of apartments in last db-inserted offering.
"""
db_conn_status = db_connection.is_connected()
if not db_conn_status:
db_connection.connect()
try:
size = db_connection.get_current_offer_size()
return size
except DatabaseException as e:
print(str(e))
finally:
if not db_conn_status:
db_connection.disconnect()