def test_expirable_redis_hash_dict(self):
"Test the redis hash dict implementation."
hash_key = "%s.hash_dict" % self.prefix
key = "hello"
# ensure dictionary isn't here
rd = RedisDict()
del rd[hash_key]
for class_impl in (ExpirableRedisHashDict, ExpirablePickleRedisHashDict, ExpirableJSONRedisHashDict):
rhd = class_impl(hash_key)
rhd.set_default_expiration(0.1)
self.assertEqual(len(rhd), 0)
self.assertFalse(key in rhd)
rhd[key] = 10
self.assertTrue(key in rhd)
self.assertEqual(len(rhd), 1)
# pass through serialize loses type information, whereas
# the other serializers retain type correctly, hence the
# ambiguity in this test
self.assertTrue(rhd[key] in ('10', 10))
del rhd[key]
self.assertFalse(key in rhd)
self.assertEqual(len(rhd), 0)
rhd[key] = 100
self.assertTrue(key in rhd)
time.sleep(0.101)
self.assertTrue(key not in rhd)
del rhd[key]
def test_redis_set(self):
"Test redis set."
set_key = "%s.list" % self.prefix
# ensure set isn't here
rd = RedisDict()
del rd[set_key]
for class_impl in (RedisSet, PickleRedisSet, JSONRedisSet):
rs = class_impl(set_key)
self.assertEquals(len(rs), 0)
rs.add("a")
rs.add("a")
rs.update(("a", "b"))
self.assertEquals(len(rs), 2)
self.assertTrue(rs.pop() in ("a", "b"))
self.assertEquals(len(rs), 1)
self.assertTrue(rs.pop() in ("a", "b"))
self.assertEquals(len(rs), 0)
def test_redis_list(self):
"Test the redis hash dict implementation."
list_key = "%s.list" % self.prefix
# ensure list isn't here
rd = RedisDict()
del rd[list_key]
for class_impl in (RedisList, PickleRedisList, JSONRedisList):
rl = class_impl(list_key)
self.assertEqual(len(rl), 0)
rl.append("a")
rl.append("b")
self.assertEqual(len(rl), 2)
self.assertEquals(rl[0], "a")
self.assertEquals(rl[-1], "b")
self.assertEquals(rl[:1], ["a", "b"])
self.assertEquals(rl[:], ["a", "b"])
self.assertEquals(rl.pop(), "b")
self.assertEquals(rl.pop(), "a")
import json
import random
from loguru import logger as log
from redis_dict import RedisDict
import pandas as pd
from anton.honestdoor_utils import login, get_page_source, read_transaction_table, read_assessment_price
import os
# Get a subset from Redis
namespace = 'house-search'
NSUB = 1000
r_dic = RedisDict(namespace=namespace, host="10.30.40.132")
listings_all = r_dic.redis.smembers('house-search:listings')
listings_honestdoor = r_dic.redis.smembers('house-search:listings_honestdoor')
listings_honestdoor_blacklist = r_dic.redis.smembers(
'house-search:listings_honestdoor_blacklist')
log.info(
f'Listings processed: {len(listings_honestdoor)} | total: {len(listings_all)} | blacklisted: {len(listings_honestdoor_blacklist)}'
)
listings_difference = list(listings_all - listings_honestdoor -
listings_honestdoor_blacklist)
log.info(f'Difference size: {len(listings_difference)}')
listings_to_process = random.choices(listings_difference, k=NSUB)
dict_to_process = {'listing': listings_to_process}
with open('listings.json', 'w') as f:
json.dump(dict_to_process, f, indent=4)
# Run transaction pulling
from dotenv import load_dotenv, find_dotenv
from loguru import logger as log
from redis_dict import RedisDict
from anton.honestdoor_utils import login, read_transaction_table, get_page_source, read_assessment_price
import pandas as pd
import os
load_dotenv(find_dotenv())
driver = login()
pages_to_parse = pd.read_json('listings.json')
tmp_data_folder = os.environ['TMP_DATA']
os.makedirs(tmp_data_folder, exist_ok=True)
namespace = 'house-search'
r_dic = RedisDict(namespace=namespace)
listings_all = r_dic.redis.smembers(f'{namespace}:listings')
listings_honestdoor = r_dic.redis.smembers(f'{namespace}:listings_honestdoor')
for property in pages_to_parse['listing']:
prop_url_part = property.split('-calgary')[0] + '-calgary-ab'
full_url = f'https://www.honestdoor.com/property/{prop_url_part}'
fname = os.path.join(tmp_data_folder, f'{property}.json')
log.info(f'Processing {full_url}')
try:
page_str = get_page_source(full_url, driver)
if not ('page could not be found' in page_str):
df = read_transaction_table(page_str, property)
city_price = read_assessment_price(page_str)
df['assessment_price'] = city_price
df.to_json(fname)
def create_redis_dict(cls, namespace=TEST_NAMESPACE_PREFIX, **kwargs):
config = redis_config.copy()
config.update(kwargs)
return RedisDict(namespace=namespace, **config)
import time
from redis_dict import RedisDict
d = RedisDict(namespace='app_name2')
assert 'random' not in d
d['random'] = 4
assert d['random'] == 4
assert 'random' in d
del d['random']
assert 'random' not in d
deep = ['key', 'key1', 'key2']
deep_val = 'mister'
d.chain_set(deep, deep_val)
assert deep_val == d.chain_get(deep)
d.chain_del(deep)
try:
d.chain_get(deep)
except KeyError:
pass
except Exception:
print('failed to throw KeyError')
else:
print('failed to throw KeyError')
assert 'random' not in d
d['random'] = 4
dd = RedisDict(namespace='app_name_too')
assert len(dd) == 0
import holoviews as hv
import numpy as np
import pandas as pd
import panel as pn
import param
from bokeh.models import HoverTool, ResetTool, PanTool, WheelZoomTool, ColumnDataSource
from bokeh.plotting import figure
from holoviews.util.transform import lon_lat_to_easting_northing, easting_northing_to_lon_lat
from redis_dict import RedisDict
from tqdm import tqdm
from loguru import logger as log
from constants import CSS_CLASS_CARD
from utils import get_price_range, OSM_tile_source
from bokeh.models.widgets.tables import HTMLTemplateFormatter, NumberFormatter
r_dic = RedisDict(namespace='house-search', host="10.30.40.132")
bootstrap = pn.template.BootstrapTemplate(title='Smart House Search')
pn.config.sizing_mode = "stretch_width"
pn.extension(raw_css=[CSS_CLASS_CARD])
TOOLTIPS = """
<div width="520" style="width:350px;">
<div>
<img
src="@photo" height="128" alt="@photo" width="128"
style="float: left; margin: 0px 15px 15px 0px;"
border="2"
></img>
</div>
<div>
<span style="font-size: 12px; font-weight: bold;">@address</span>