def test_empty(self):
listings1 = listing.Listings()
self.assertEqual(0, len(listings1), 'listing1 not empty at beginning')
listings2 = listing.Listings()
self.assertEqual(0, len(listings2), 'listing2 not empty at beginning')
update = listings1.update(listings2)
self.assertEqual(0, len(update), 'update is not empty')
self.assertEqual(0, len(listings1), 'listing1 not in original state')
def test_empty(self):
listings1 = listing.Listings()
listings1_len = len(listings1)
self.assertEqual(0, listings1_len, 'listing1 not empty at beginning')
listings2 = listing.Listings()
self.assertEqual(0, len(listings2), 'listing2 not empty at beginning')
removed = listings1.remove_not_existing(listings2)
self.assertEqual(0, len(removed),
'number of removed entries does not match')
self.assertEqual(listings1_len - len(removed), len(listings1),
'remaining number of entries does not match')
def read(file_name):
file = pathlib.Path(file_name)
if file.exists():
data = pickle.loads(file.read_bytes())
logging.info(f"Read {len(data)} entries")
return data
return listing.Listings()
def test_update_empty(self):
entry = listing.Entry()
entry.id = 'foobar'
entry.mod_date = datetime.now()
listings1 = listing.Listings()
listings1.foo = entry
self.assertEqual(1, len(listings1),
'listing1 does not contain entry at beginning')
listings2 = listing.Listings()
self.assertEqual(0, len(listings2), 'listing2 not empty at beginning')
update = listings1.update(listings2)
self.assertEqual(0, len(update), 'update is not empty')
self.assertEqual(1, len(listings1), 'listing1 not in original state')
def test_remove(self):
entry = listing.Entry()
entry.mod_date = datetime.now()
listings1 = listing.Listings()
listings1.foo = entry
listings1_len = len(listings1)
self.assertEqual(1, listings1_len,
'listing1 does not contain entry at beginning')
listings2 = listing.Listings()
self.assertEqual(0, len(listings2), 'listing2 not empty at beginning')
removed = listings1.remove_not_existing(listings2)
self.assertEqual(1, len(removed),
'number of removed entries does not match')
self.assertEqual(listings1_len - len(removed), len(listings1),
'remaining number of entries does not match')
def test_no_overlap(self):
entry1 = listing.Entry()
entry1.id = 'foobar1'
entry1.mod_date = datetime.now()
listings1 = listing.Listings()
listings1.foo1 = entry1
self.assertEqual(1, len(listings1),
'listing1 does not contain entry at beginning')
entry2 = listing.Entry()
entry2.id = 'foobar2'
entry2.mod_date = datetime.now()
listings2 = listing.Listings()
listings2.foo2 = entry2
self.assertEqual(1, len(listings2),
'listing2 does not contain entry at beginning')
update = listings1.update(listings2)
self.assertEqual(1, len(update), 'update should contain one entry')
self.assertEqual(2, len(listings1),
'listing1 does not contain both entries')
def get_apartments(config):
ignore = ['listing', '__init__']
apartments = listing.Listings()
for child in pwd.iterdir():
if child.is_file(
) and child.stem not in ignore and child.suffix == '.py' and not child.stem.startswith(
"test_"):
name = child.stem
imported = getattr(
__import__('immocrawler.provider', fromlist=[name]), name)
res = imported.get_apartments(config[name])
apartments.update(res)
return apartments
def test_one_update(self):
entry1 = listing.Entry()
entry1.id = 'foobar1'
entry1.mod_date = datetime.now()
listings1 = listing.Listings()
listings1.foo = entry1
self.assertEqual(1, len(listings1),
'listing1 does not contain entry at beginning')
entry2 = listing.Entry()
entry2.id = 'foobar2'
entry2.mod_date = datetime.now()
listings2 = listing.Listings()
listings2.foo = entry2
self.assertEqual(1, len(listings2),
'listing1 does not contain entry at beginning')
update = listings1.update(listings2)
self.assertEqual(1, len(update), 'update should contain one entry')
self.assertEqual(1, len(listings1),
'listings1 should contain one entry')
self.assertEqual(entry2.id, listings1.foo.id)
def _get_apartments(state, city):
i = 1
results = listing.Listings()
while True:
url = f'https://www.immobilienscout24.de/Suche/S-T/P-{i}/Wohnung-Miete/{state}/{city}/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/-/2,3'
r = requests.post(url)
result_list = r.json()['searchResponseModel']['resultlist.resultlist']
entries = result_list['resultlistEntries'][0]['resultlistEntry']
for entry in entries:
try:
entry = __make_entry(entry)
except Exception as e:
logger.error(f'failed to create entry: {e}')
else:
setattr(results, entry.uuid, entry)
i += 1
if 'next' not in result_list['paging']:
break
return results