def test_one_survey_needs_updates_no_homes_valid(self, mock_os, mock_alog):
"""
Tests that if a survey needs updating but there isn't enough homes that meet
the criteria then the email is not called
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.APARTMENT)[0]
survey = RentingSurveyModel.create_survey(user.userProfile,
home_type=home_type,
wants_update=True,
update_frequency=0,
score_threshold=100,
num_home_threshold=3)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
# Create homes that will return from the algorithm
home = HomeScore(home)
home.accumulated_points = 100
home.total_possible_points = 100
home1 = HomeScore(home1)
home1.accumulated_points = 50
home1.total_possible_points = 100
mc = mock_alog.return_value
mc.homes = [home, home1]
# Act
notify_user_survey_updates()
# Assert
mock_os.assert_not_called()
def test_survey_marked_for_no_update(self, mock_os, mock_alog):
"""
Tests that if the user marks to not update the survey then the survey is not updated
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.APARTMENT)[0]
survey = RentingSurveyModel.create_survey(user.userProfile,
home_type=home_type,
wants_update=False,
update_frequency=0,
score_threshold=100,
num_home_threshold=1)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
# Create homes that will return from the algorithm
home = HomeScore(home)
home.accumulated_points = 100
home.total_possible_points = 100
home1 = HomeScore(home1)
home1.accumulated_points = 50
home1.total_possible_points = 100
mc = mock_alog.return_value
mc.homes = [home, home1]
# Act
notify_user_survey_updates()
# Assert
mock_os.assert_not_called()
def test_determine_threshold_trigger_is_triggered_all_homes(self):
"""
Tests if an email should be trigger based on the user criteria
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile,
num_home_threshold=3,
score_threshold=70)
home = HomeScore(RentDatabaseModel.create_house_database())
home1 = HomeScore(RentDatabaseModel.create_house_database())
home2 = HomeScore(RentDatabaseModel.create_house_database())
# Give each home a score of 100
home.accumulated_points = 50
home.total_possible_points = 50
home1.accumulated_points = 50
home1.total_possible_points = 50
home2.accumulated_points = 50
home2.total_possible_points = 50
# Act
result = survey.determine_threshold_trigger([home, home1, home2])
# Assert
self.assertEqual(result, [home, home1, home2])
def test_one_survey_needs_updates(self, mock_os, mock_alog):
"""
Tests that if at least one home has past the score threshold and the survey update timestamp
is valid, then the survey updates and calls the email function
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.APARTMENT)[0]
survey = RentingSurveyModel.create_survey(user.userProfile,
home_type=home_type,
wants_update=True,
update_frequency=0,
score_threshold=100,
num_home_threshold=1)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
# Create homes that will return from the algorithm
home = HomeScore(home)
home.accumulated_points = 100
home.total_possible_points = 100
home1 = HomeScore(home1)
home1.accumulated_points = 50
home1.total_possible_points = 100
mc = mock_alog.return_value
mc.homes = [home, home1]
# Act
notify_user_survey_updates()
# Assert
mock_os.assert_called_once_with(survey, 1)
def tests_check_home_in_blacklist_is_not_in_blacklist(self):
"""
Tests that if a home is not in the blacklist then the function returns false
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
survey.blacklisted_homes.add(home)
# Act
result = survey.check_home_in_blacklist(home1)
# Assert
self.assertFalse(result)
def test_blacklist_home_homes_already_exist(self):
"""
Tests adding a home when a home already exists and makes sure both homes are in the list after wards
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
survey.blacklisted_homes.add(home)
# Act
survey.blacklist_home(home1)
# Assert
self.assertEqual(survey.blacklisted_homes.count(), 2)
self.assertTrue(survey.blacklisted_homes.filter(id=home.id).exists())
self.assertTrue(survey.blacklisted_homes.filter(id=home1.id).exists())
def test_multiple_surveys_one_wants_updating(self, mock_os, mock_alog):
"""
Tests that if there are multiple surveys but one is marked as want update,
then only that survey gets updated
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.APARTMENT)[0]
survey = RentingSurveyModel.create_survey(user.userProfile,
home_type=home_type,
wants_update=False,
update_frequency=0,
score_threshold=100,
num_home_threshold=1)
survey1 = RentingSurveyModel.create_survey(user.userProfile,
home_type=home_type,
wants_update=True,
update_frequency=0,
score_threshold=100,
num_home_threshold=1)
home = RentDatabaseModel.create_house_database()
home1 = RentDatabaseModel.create_house_database()
# Create homes that will return from the algorithm
home = HomeScore(home)
home.accumulated_points = 100
home.total_possible_points = 100
home1 = HomeScore(home1)
home1.accumulated_points = 50
home1.total_possible_points = 100
mc = mock_alog.return_value
mc.homes = [home, home1]
# Act
notify_user_survey_updates()
# Assert
mock_os.assert_has_calls([call(survey1, 1)])
def test_determine_threshold_trigger_homes_in_blacklist_do_not_trigger(
self):
"""
Tests that if homes are in the blacklist then they are not counted as part of the trigger criteria.
This specifically tests that if there was 5 homes that fit but 3 are in the blacklist then
the criteria is not hit and thus returns empty list
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile,
num_home_threshold=3,
score_threshold=70)
home = HomeScore(RentDatabaseModel.create_house_database())
home1 = HomeScore(RentDatabaseModel.create_house_database())
home2 = HomeScore(RentDatabaseModel.create_house_database())
home3 = HomeScore(RentDatabaseModel.create_house_database())
home4 = HomeScore(RentDatabaseModel.create_house_database())
survey.blacklist_home(home.home)
survey.blacklist_home(home2.home)
survey.blacklist_home(home3.home)
# Give each home a score of 100
home.accumulated_points = 50
home.total_possible_points = 50
home1.accumulated_points = 50
home1.total_possible_points = 50
home2.accumulated_points = 50
home2.total_possible_points = 50
home3.accumulated_points = 50
home3.total_possible_points = 50
home4.accumulated_points = 50
home4.total_possible_points = 50
# Act
result = survey.determine_threshold_trigger(
[home, home1, home2, home3, home4])
# Assert
self.assertEqual(result, [])
def test_determine_threshold_trigger_homes_in_blacklist_not_returned(self):
"""
Tests that if there are still enough homes after the blacklist homes are removed then
homes not in the blacklist are returned
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile,
num_home_threshold=3,
score_threshold=70)
home = HomeScore(RentDatabaseModel.create_house_database())
home1 = HomeScore(RentDatabaseModel.create_house_database())
home2 = HomeScore(RentDatabaseModel.create_house_database())
home3 = HomeScore(RentDatabaseModel.create_house_database())
home4 = HomeScore(RentDatabaseModel.create_house_database())
survey.blacklist_home(home.home)
survey.blacklist_home(home3.home)
# Give each home a score of 100
home.accumulated_points = 50
home.total_possible_points = 50
home1.accumulated_points = 50
home1.total_possible_points = 50
home2.accumulated_points = 50
home2.total_possible_points = 50
home3.accumulated_points = 50
home3.total_possible_points = 50
home4.accumulated_points = 50
home4.total_possible_points = 50
# Act
result = survey.determine_threshold_trigger(
[home, home1, home2, home3, home4])
# Assert
self.assertEqual(result, [home1, home2, home4])
def tests_check_home_in_blacklist_black_list_empty(self):
"""
Tests that if the black list is empty the function returns false
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile)
home = RentDatabaseModel.create_house_database()
# Act
result = survey.check_home_in_blacklist(home)
# Assert
self.assertFalse(result)
def test_blacklisting_homes(self):
"""
Tests just adding a home to the blacklisted_homes list and making sure it exists in field
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile)
home = RentDatabaseModel.create_house_database()
# Act
survey.blacklist_home(home)
# Assert
self.assertEqual(survey.blacklisted_homes.count(), 1)
self.assertTrue(survey.blacklisted_homes.filter(id=home.id).exists())
def tests_adding_same_home_does_not_add_duplicate(self):
"""
Tests that adding a duplicate home will not cause 2 homes to be in the list
"""
# Arrange
user = MyUser.objects.create(email="*****@*****.**")
survey = RentingSurveyModel.create_survey(user.userProfile)
home = RentDatabaseModel.create_house_database()
survey.blacklisted_homes.add(home)
# Act
survey.blacklist_home(home)
# Assert
self.assertEqual(survey.blacklisted_homes.count(), 1)
self.assertTrue(survey.blacklisted_homes.filter(id=home.id).exists())
def parse_idx_feed(self):
root = self.ygl_file.getroot()
# Reporting information
num_houses = 0
num_of_duplicates = 0
num_of_value_errors = 0
num_integrity_error = 0
num_updated_homes = 0
num_added_homes = 0
# Loop through every home
for house in root.iter('Rental'):
num_houses += 1
new_listing = RentDatabaseModel()
street_number = ""
street_name = ""
for element in house:
try:
if element.tag == 'ID':
new_listing.listing_number = element.text
elif element.tag == 'StreetNumber':
street_number = element.text
elif element.tag == 'StreetName':
street_name = element.text
elif element.tag == 'City':
new_listing.city = element.text
elif element.tag == 'State':
new_listing.state = element.text
elif element.tag == 'Zip':
new_listing.zip_code = element.text
elif element.tag == 'UnitNumber':
new_listing.apartment_number = element.text.lower()
elif element.tag == 'Latitude':
new_listing.latitude = element.text
elif element.tag == 'Longitude':
new_listing.longitude = element.text
elif element.tag == 'Beds':
new_listing.num_bedrooms = int(element.text)
elif element.tag == 'Baths':
# don't support decimals right now
new_listing.num_bathrooms = round(float(element.text))
elif element.tag == 'AvailableDate':
date_available = datetime.strptime(element.text, '%m/%d/%Y')
new_listing.date_available = date_available
# Need to compare non-naive timezone date to non-naive.
# This way the dates are comparable
date_available = pytimezone('US/Eastern').localize(date_available)
if timezone.now() > date_available - timedelta(days=CURRENTLY_AVAILABLE_DELTA_DAYS):
new_listing.currently_available = True
else:
new_listing.currently_available = False
elif element.tag == 'Pet':
if 'Dog Ok' in element.text:
new_listing.dogs_allowed = True
elif 'Cat Ok' in element.text:
new_listing.cats_allowed = True
elif 'Pet Friendly' in element.text:
new_listing.dogs_allowed = True
new_listing.cats_allowed = True
elif 'Negotiable' in element.text:
new_listing.dogs_allowed = True
new_listing.cats_allowed = True
elif element.tag == 'Parking':
if element.text == 'Included':
new_listing.parking_spot = True
elif element.tag == 'Price':
new_listing.price = int(element.text)
elif element.tag == 'Features':
# Initialize word scraper
word_scraper = WordScraper(element.text)
new_listing.laundromat_nearby = word_scraper.look_for_laundromat()
new_listing.furnished = word_scraper.look_for_furnished()
new_listing.hardwood_floors = word_scraper.look_for_hardwood_floors()
new_listing.dishwasher = word_scraper.look_for_dishwasher()
new_listing.air_conditioning = word_scraper.look_for_ac()
new_listing.pool = word_scraper.look_for_pool()
new_listing.patio_balcony = word_scraper.look_for_balcony()
new_listing.laundry_in_building = word_scraper.look_for_laundry_in_building()
new_listing.laundry_in_unit = word_scraper.look_for_laundry_in_unit()
new_listing.gym = word_scraper.look_for_gym()
new_listing.storage = word_scraper.look_for_storage()
new_listing.remarks = element.text
except ValueError:
print("[ VALUE ERROR ] Could not add home")
num_of_value_errors += 1
continue
new_listing.home_type = HomeTypeModel.objects.get(home_type=HomeTypeModel.APARTMENT)
new_listing.listing_provider = HomeProviderModel.objects.get(provider="YGL")
new_listing.last_updated = self.update_timestamp
new_listing.street_address = normalize_street_address("{0} {1}".format(street_number, street_name))
# Determines if the home already exists as a YGL house
if RentDatabaseModel.objects\
.filter(listing_provider=new_listing.listing_provider) \
.filter(street_address=new_listing.street_address)\
.filter(city=new_listing.city) \
.filter(state=new_listing.state)\
.filter(zip_code=new_listing.zip_code)\
.filter(apartment_number=new_listing.apartment_number)\
.exists():
# Retrieve the home that the home matches
existing_apartment = RentDatabaseModel.objects.get(
street_address=new_listing.street_address,
city=new_listing.city,
state=new_listing.state,
zip_code=new_listing.zip_code,
apartment_number=new_listing.apartment_number
)
existing_apartment.update(new_listing)
existing_apartment.save()
num_updated_homes += 1
print("[ UPDATED ] {0}".format(existing_apartment.full_address))
# Tests if the home exists within another provider
# If so mark it as a duplicate and don't add it
elif RentDatabaseModel.objects\
.filter(street_address=new_listing.street_address) \
.filter(city=new_listing.city) \
.filter(state=new_listing.state) \
.filter(zip_code=new_listing.zip_code)\
.filter(apartment_number=new_listing.apartment_number)\
.exists():
num_of_duplicates += 1
print("[ DUPLICATE ] " + new_listing.full_address)
else:
try:
new_listing.save()
print("[ ADDING ] " + new_listing.full_address)
num_added_homes += 1
except IntegrityError:
print("[ Integrity Error ] ")
num_integrity_error += 1
manager = HomeProviderModel.objects.get(provider="YGL")
manager.last_updated_feed = self.update_timestamp
manager.save()
print("")
print("RESULTS:")
logger.info("\nNumber of houses in database: {0}\n".format(num_houses) +
"Num added homes: {0}\n".format(num_added_homes) +
"Num updated homes: {0}\n".format(num_updated_homes) +
"Update timestamp: {0}\n".format(self.update_timestamp.date()) +
"Number of duplicates: {0}\n".format(num_of_duplicates) +
"Number of value errors: {0}\n".format(num_of_value_errors) +
"Number of integrity error is: {0}\n".format(num_integrity_error))
def add_home_to_database(self, home):
new_listing = RentDatabaseModel()
num_of_value_errors = 0
try:
# Home Address info
new_listing.street_address = "{0} {1}".format(
home['StreetNumber'], home['StreetName']).replace(',', '')
new_listing.city = home['City']
new_listing.state = home['StateOrProvince']
new_listing.zip_code = home['PostalCode']
new_listing.latitude = home['Latitude']
new_listing.longitude = home['Longitude']
# Home Basic info
new_listing.price = int(float(home['ListPrice']))
new_listing.num_bedrooms = int(home['BedroomsTotal'])
new_listing.num_bathrooms = int(home['BathroomsFull'])
new_listing.apartment_number = home['UnitNumber']
# MLS listing information
new_listing.remarks = home['PublicRemarks']
new_listing.listing_number = home['ListingId']
new_listing.listing_agent = home['ListAgentMlsId']
new_listing.listing_office = home['ListOfficeMlsId']
new_listing.listing_provider = HomeProviderModel.objects.get_or_create(
provider=HomeProviderModel.MLSPIN)[0]
new_listing.showing_instructions = home['ShowingInstructions']
new_listing.showing_remarks = home['FIRM_RMK1']
# Amenities
new_listing.dogs_allowed = 'yes' in home['PETS_ALLOWED'].lower()
new_listing.cats_allowed = 'yes' in home['PETS_ALLOWED'].lower()
word_scraper_remarks = WordScraper(new_listing.remarks)
word_scraper_appliances = WordScraper(home['Appliances'])
new_listing.air_conditioning = home['AIR_CONDITION'] == 'Yes'
if word_scraper_remarks.look_for_ac(
) or word_scraper_appliances.look_for_ac():
new_listing.air_conditioning = True
new_listing.furnished = word_scraper_remarks.look_for_furnished() \
or word_scraper_appliances.look_for_furnished()
new_listing.hardwood_floors = word_scraper_remarks.look_for_hardwood_floors() \
or word_scraper_appliances.look_for_hardwood_floors()
new_listing.dishwasher = word_scraper_remarks.look_for_dishwasher() \
or word_scraper_appliances.look_for_dishwasher()
new_listing.laundry_in_building = word_scraper_remarks.look_for_laundry_in_building() \
or word_scraper_appliances.look_for_laundry_in_building()
new_listing.pool = word_scraper_remarks.look_for_pool()
new_listing.patio_balcony = word_scraper_remarks.look_for_balcony()
new_listing.storage = word_scraper_remarks.look_for_storage()
new_listing.last_updated = self.update_timestamp
list_type = home['RN_TYPE']
if list_type == "Apartment":
new_listing.home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.APARTMENT)[0]
elif list_type == "Single Family":
new_listing.home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.SINGLE_FAMILY)[0]
elif list_type == "Condominium":
new_listing.home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.CONDO)[0]
else:
new_listing.home_type = HomeTypeModel.objects.get_or_create(
home_type=HomeTypeModel.OTHER)[0]
if home['Date_Available']:
date_available = datetime.strptime(home['Date_Available'],
'%Y-%m-%dT%H:%M:%S')
new_listing.date_available = date_available
date_available = pytimezone('US/Eastern').localize(
date_available)
if timezone.now() > date_available - timedelta(
days=CURRENTLY_AVAILABLE_DELTA_DAYS):
new_listing.currently_available = True
else:
self.num_available_in_future += 1
else:
new_listing.currently_available = True
except ValueError:
num_of_value_errors += 1
return
# Determines if the home already exists as a MLSPIN house
if RentDatabaseModel.objects \
.filter(listing_provider=new_listing.listing_provider) \
.filter(street_address=new_listing.street_address) \
.filter(city=new_listing.city) \
.filter(state=new_listing.state) \
.filter(zip_code=new_listing.zip_code) \
.filter(apartment_number=new_listing.apartment_number) \
.exists():
# Retrieve the home that the home matches
existing_apartment = RentDatabaseModel.objects.get(
street_address=new_listing.street_address,
city=new_listing.city,
state=new_listing.state,
zip_code=new_listing.zip_code,
apartment_number=new_listing.apartment_number)
# Since the apartments are the same
# Update the existing apartment with the fields stored in the new listing
existing_apartment.update(new_listing)
try:
existing_apartment.save()
print("[ UPDATED ] {0}".format(
existing_apartment.full_address))
self.num_updated_homes += 1
except ValidationError:
print('Validation error')
self.num_validation_error += 1
# Tests if the home exists within another provider
# If so mark it as a duplicate and don't add it
elif RentDatabaseModel.objects \
.filter(street_address=new_listing.street_address) \
.filter(city=new_listing.city) \
.filter(state=new_listing.state) \
.filter(zip_code=new_listing.zip_code) \
.filter(apartment_number=new_listing.apartment_number) \
.exists():
print("[ DUPLICATE ] {0}".format(new_listing.full_address))
self.num_of_duplicates += 1
else:
try:
new_listing.save()
self.num_added_homes += 1
print("[ ADDING ] " + new_listing.full_address)
except IntegrityError:
print("[ Integrity Error ] ")
self.num_integrity_errors += 1
except ValidationError:
print("[ Validation Error ] ")
self.num_validation_error += 1
def test_valid_update(self):
# Arrange
# Create a home and save it
dest = RentDatabaseModel(
apartment_number='1r',
street_address='12 Stony Brook Rd',
city='Arlington',
state='MA',
zip_code='02476',
latitude=25.25,
longitude=24.24,
num_bathrooms=2,
num_bedrooms=1,
parking_spot=True,
remarks="hi",
listing_number='123',
listing_provider=HomeProviderModel.objects.get(provider="MLSPIN"),
listing_agent='The agent',
listing_office='The office',
last_updated=timezone.now(),
price=2000,
home_type=self.home_type,
currently_available=True,
)
dest.save()
dest_id = dest.id
# Create another home to update the old home with
dest1 = RentDatabaseModel(
apartment_number='3f',
street_address='360 Huntington Ave',
city='Boston',
state='RI',
zip_code='02474',
latitude=23.23,
longitude=22.22,
num_bathrooms=3,
num_bedrooms=2,
parking_spot=False,
remarks="the",
listing_number='456',
listing_provider=HomeProviderModel.objects.get(provider="YGL"),
listing_agent='The agent 2',
listing_office='The office 2',
last_updated=timezone.now() + timezone.timedelta(days=1),
price=2400,
home_type=self.home_type1,
currently_available=False,
)
# Act
# call the update function
dest.update(dest1)
dest.save()
# Assert values of dest in database
home = RentDatabaseModel.objects.get(id=dest_id)
# Make sure the update function does not create a new model
self.assertEqual(RentDatabaseModel.objects.count(), 1)
# Asserts that every field can be updated
self.assertEqual(home.apartment_number, '3f')
self.assertEqual(home.street_address, '360 Huntington Ave')
self.assertEqual(home.city, 'Boston')
self.assertEqual(home.state, 'RI')
self.assertEqual(home.zip_code, '02474')
self.assertAlmostEquals(home.latitude, Decimal(23.23))
self.assertAlmostEquals(home.longitude, Decimal(22.22))
self.assertEqual(home.num_bathrooms, 3)
self.assertEqual(home.num_bedrooms, 2)
self.assertFalse(home.parking_spot)
self.assertEqual(home.remarks, 'the')
self.assertEqual(home.listing_number, 456)
self.assertEqual(home.listing_provider,
HomeProviderModel.objects.get(provider="YGL"))
self.assertEqual(home.listing_agent, 'The agent 2')
self.assertEqual(home.listing_office, 'The office 2')
self.assertEqual(home.last_updated,
(timezone.now() + timezone.timedelta(days=1)).date())
self.assertEqual(home.price, 2400)
self.assertEqual(home.home_type, self.home_type1)
self.assertFalse(home.currently_available)
def parse_idx_feed(self):
lines = self.idx_txt
print("Attempting to add *" + str(len(lines)) +
"* apartments to the db...")
print("An equivalent number of requests will be made to the geocoder")
# Generate values for the different error cases for tracking purposes
num_houses = 0
num_of_duplicates = 0
num_of_value_errors = 0
num_failed_to_update = 0
num_failed_to_geolocate = 0
num_not_for_rental = 0
num_integrity_error = 0
num_added_homes = 0
num_updated_homes = 0
num_homes_not_enough_cells = 0
counter = 0
for line in lines[1:]: # skips the col headers
# if self.num_homes is equal to -1, then it means to loop through all homes,
# otherwise just loop for the indicated number of homes
if self.num_homes != -1 and counter >= self.num_homes:
break
counter = counter + 1
num_houses += 1
new_listing = RentDatabaseModel()
# Parse IDX feed to put each item into an array
cells = line.split('|')
# If the home doesn't have enough cells then something is wrong with the listing and it won't
# be added to the database. Otherwise it will cause an exception
if len(cells) < 28:
print("Removing home not enough cells")
num_homes_not_enough_cells += 1
continue
# Make sure there are no commas in the street name
cells[STREET_NAME].replace(',', '')
split_address = cells[STREET_NAME].split()
# Needed Variables
clean_address = ""
try:
# check for presence of apartment number with int()
int(cells[STREET_NAME][len(cells[STREET_NAME]) - 1])
# the purpose of encoding and then decoding is to remove any non-ascii characters
clean_address = " ".join(split_address[:-1]).encode(
'ascii', errors='ignore').decode()
# no int in last address element (not an apartment #)
except ValueError:
clean_address = " ".join(split_address).encode(
'ascii', errors='ignore').decode()
# If any of the fields give a value error, then don't save the apartment
try:
# Initialize word scraper
word_scraper = WordScraper(cells[REMARKS])
# Set the HomeBaseModel Fields
new_listing.street_address = normalize_street_address(
"{0} {1}".format(cells[STREET_NO], clean_address))
new_listing.city = self.towns[str(cells[TOWN_NUM])]["town"]
new_listing.state = self.towns[str(cells[TOWN_NUM])]["state"]
new_listing.zip_code = cells[ZIP_CODE]
new_listing.price = int(cells[LIST_PRICE])
new_listing.laundromat_nearby = word_scraper.look_for_laundromat(
)
# Set InteriorAmenitiesModel Fields
# Currently don't support non-integers for num_bathrooms. Therefore
# The num of full and half baths are added then rounded to the nearest int
num_baths = int(cells[NO_FULL_BATHS]) + int(
cells[NO_HALF_BATHS])
new_listing.bath = True if num_baths > 0 else False
new_listing.num_bathrooms = num_baths
new_listing.num_bedrooms = int(cells[NO_BEDROOMS])
new_listing.furnished = word_scraper.look_for_furnished()
new_listing.hardwood_floors = word_scraper.look_for_hardwood_floors(
)
new_listing.dishwasher = word_scraper.look_for_dishwasher()
new_listing.air_conditioning = word_scraper.look_for_ac()
new_listing.dogs_allowed = word_scraper.look_for_pets("dogs")
new_listing.cats_allowed = word_scraper.look_for_pets("cats")
new_listing.laundry_in_building = word_scraper.look_for_laundry_in_building(
)
# Set MLSpinDataModel fields
new_listing.remarks = cells[REMARKS]
new_listing.listing_number = int(cells[LIST_NO])
new_listing.listing_provider = HomeProviderModel.objects.get(
provider=HomeProviderModel.MLSPIN)
new_listing.listing_agent = cells[LIST_AGENT]
new_listing.listing_office = cells[LIST_OFFICE]
new_listing.last_updated = self.update_timestamp
# Set RentDatabaseModel fields
new_listing.apartment_number = cells[UNIT_NO].lower()
# Set Exterior Amenities fields
if int(cells[PARKING_SPACES]) > 0:
new_listing.parking_spot = True
new_listing.pool = word_scraper.look_for_pool()
new_listing.patio_balcony = word_scraper.look_for_balcony()
new_listing.laundry_in_unit = word_scraper.look_for_laundry_in_unit(
)
new_listing.gym = word_scraper.look_for_gym()
new_listing.storage = word_scraper.look_for_storage()
# Create the new home
# Define the home type
list_type = cells[PROP_TYPE]
# verifies unit is a rental (RN denotes rental in MLS feed)
if list_type == "RN":
apartment_home_type = HomeTypeModel.objects.get(
home_type=HomeTypeModel.APARTMENT)
else:
# Since we only support rentals right now we don't want to retrieve any other home types
print(
"Home not a rental, continuing. Error was with line {0}"
.format(line))
num_not_for_rental += 1
continue
new_listing.home_type = apartment_home_type
new_listing.currently_available = True
except ValueError:
print(
"Home could not be added. Error is with line: {0}".format(
line))
num_of_value_errors += 1
continue
# Determines if the home already exists as a MLSPIN house
if RentDatabaseModel.objects \
.filter(listing_provider=new_listing.listing_provider) \
.filter(street_address=new_listing.street_address) \
.filter(city=new_listing.city) \
.filter(state=new_listing.state) \
.filter(zip_code=new_listing.zip_code) \
.filter(apartment_number=new_listing.apartment_number) \
.exists():
# Retrieve the home that the home matches
existing_apartment = RentDatabaseModel.objects.get(
street_address=new_listing.street_address,
city=new_listing.city,
state=new_listing.state,
zip_code=new_listing.zip_code,
apartment_number=new_listing.apartment_number)
# The lat and long is the only thing that is not computed for each new_listing since it costs money
# Therefore assume the old lat and long values are correct (Should not change)
new_listing.latitude = existing_apartment.latitude
new_listing.longitude = existing_apartment.longitude
# Since the apartments are the same
# Update the existing apartment with the fields stored in the new listing
existing_apartment.update(new_listing)
existing_apartment.save()
print("[ UPDATED ] {0}".format(
existing_apartment.full_address))
num_updated_homes += 1
# Tests if the home exists within another provider
# If so mark it as a duplicate and don't add it
elif RentDatabaseModel.objects \
.filter(street_address=new_listing.street_address) \
.filter(city=new_listing.city) \
.filter(state=new_listing.state) \
.filter(zip_code=new_listing.zip_code) \
.filter(apartment_number=new_listing.apartment_number) \
.exists():
print("[ DUPLICATE ] {0}".format(new_listing.full_address))
num_of_duplicates += 1
else:
# If it is a new home then get the lat and long of the home.
latlng = geolocator.maps_requester(
gmaps_api_key).get_lat_lon_from_address(
new_listing.full_address)
if latlng == -1:
print(
"Could not generate Lat and Long for apartment {0}, which had line {1} in IDX feed"
.format(new_listing.full_address, line))
num_failed_to_geolocate += 1
continue
else:
lat = latlng[0]
lng = latlng[1]
new_listing.latitude = lat
new_listing.longitude = lng
# After all the data is added, save the home to the database
try:
new_listing.save()
num_added_homes += 1
print("[ ADDING ] " + new_listing.full_address)
except IntegrityError:
print("[ Integrity Error ] ")
num_integrity_error += 1
manager = HomeProviderModel.objects.get(provider="MLSPIN")
manager.last_updated_feed = self.update_timestamp
manager.save()
print("")
print("RESULTS:")
logger.info(
"\nNumber of houses in database: {0}\n".format(num_houses) +
"Num added homes: {0}\n".format(num_added_homes) +
"Num updated homes: {0}\n".format(num_updated_homes) +
"Update timestamp: {0}\n".format(self.update_timestamp.date()) +
"Number of duplicates: {0}\n".format(num_of_duplicates) +
"Number of value errors: {0}\n".format(num_of_value_errors) +
"Number of failed updated houses: {0}\n".format(
num_failed_to_update) +
"Number of failed geolocates: {0}\n".format(
num_failed_to_geolocate) +
"Number of houses not for rental: {0}\n".format(
num_not_for_rental) +
"Number of integrity error is: {0}\n".format(num_integrity_error) +
"Number of homes that don't have enough cells: {0}\n".format(
num_homes_not_enough_cells))