def test_expirable_redis_hash_dict(self): "Test the redis hash dict implementation." hash_key = "%s.hash_dict" % self.prefix key = "hello" # ensure dictionary isn't here rd = RedisDict() del rd[hash_key] for class_impl in (ExpirableRedisHashDict, ExpirablePickleRedisHashDict, ExpirableJSONRedisHashDict): rhd = class_impl(hash_key) rhd.set_default_expiration(0.1) self.assertEqual(len(rhd), 0) self.assertFalse(key in rhd) rhd[key] = 10 self.assertTrue(key in rhd) self.assertEqual(len(rhd), 1) # pass through serialize loses type information, whereas # the other serializers retain type correctly, hence the # ambiguity in this test self.assertTrue(rhd[key] in ('10', 10)) del rhd[key] self.assertFalse(key in rhd) self.assertEqual(len(rhd), 0) rhd[key] = 100 self.assertTrue(key in rhd) time.sleep(0.101) self.assertTrue(key not in rhd) del rhd[key]
def test_redis_set(self): "Test redis set." set_key = "%s.list" % self.prefix # ensure set isn't here rd = RedisDict() del rd[set_key] for class_impl in (RedisSet, PickleRedisSet, JSONRedisSet): rs = class_impl(set_key) self.assertEquals(len(rs), 0) rs.add("a") rs.add("a") rs.update(("a", "b")) self.assertEquals(len(rs), 2) self.assertTrue(rs.pop() in ("a", "b")) self.assertEquals(len(rs), 1) self.assertTrue(rs.pop() in ("a", "b")) self.assertEquals(len(rs), 0)
def test_redis_list(self): "Test the redis hash dict implementation." list_key = "%s.list" % self.prefix # ensure list isn't here rd = RedisDict() del rd[list_key] for class_impl in (RedisList, PickleRedisList, JSONRedisList): rl = class_impl(list_key) self.assertEqual(len(rl), 0) rl.append("a") rl.append("b") self.assertEqual(len(rl), 2) self.assertEquals(rl[0], "a") self.assertEquals(rl[-1], "b") self.assertEquals(rl[:1], ["a", "b"]) self.assertEquals(rl[:], ["a", "b"]) self.assertEquals(rl.pop(), "b") self.assertEquals(rl.pop(), "a")
import json import random from loguru import logger as log from redis_dict import RedisDict import pandas as pd from anton.honestdoor_utils import login, get_page_source, read_transaction_table, read_assessment_price import os # Get a subset from Redis namespace = 'house-search' NSUB = 1000 r_dic = RedisDict(namespace=namespace, host="10.30.40.132") listings_all = r_dic.redis.smembers('house-search:listings') listings_honestdoor = r_dic.redis.smembers('house-search:listings_honestdoor') listings_honestdoor_blacklist = r_dic.redis.smembers( 'house-search:listings_honestdoor_blacklist') log.info( f'Listings processed: {len(listings_honestdoor)} | total: {len(listings_all)} | blacklisted: {len(listings_honestdoor_blacklist)}' ) listings_difference = list(listings_all - listings_honestdoor - listings_honestdoor_blacklist) log.info(f'Difference size: {len(listings_difference)}') listings_to_process = random.choices(listings_difference, k=NSUB) dict_to_process = {'listing': listings_to_process} with open('listings.json', 'w') as f: json.dump(dict_to_process, f, indent=4) # Run transaction pulling
from dotenv import load_dotenv, find_dotenv from loguru import logger as log from redis_dict import RedisDict from anton.honestdoor_utils import login, read_transaction_table, get_page_source, read_assessment_price import pandas as pd import os load_dotenv(find_dotenv()) driver = login() pages_to_parse = pd.read_json('listings.json') tmp_data_folder = os.environ['TMP_DATA'] os.makedirs(tmp_data_folder, exist_ok=True) namespace = 'house-search' r_dic = RedisDict(namespace=namespace) listings_all = r_dic.redis.smembers(f'{namespace}:listings') listings_honestdoor = r_dic.redis.smembers(f'{namespace}:listings_honestdoor') for property in pages_to_parse['listing']: prop_url_part = property.split('-calgary')[0] + '-calgary-ab' full_url = f'https://www.honestdoor.com/property/{prop_url_part}' fname = os.path.join(tmp_data_folder, f'{property}.json') log.info(f'Processing {full_url}') try: page_str = get_page_source(full_url, driver) if not ('page could not be found' in page_str): df = read_transaction_table(page_str, property) city_price = read_assessment_price(page_str) df['assessment_price'] = city_price df.to_json(fname)
def create_redis_dict(cls, namespace=TEST_NAMESPACE_PREFIX, **kwargs): config = redis_config.copy() config.update(kwargs) return RedisDict(namespace=namespace, **config)
import time from redis_dict import RedisDict d = RedisDict(namespace='app_name2') assert 'random' not in d d['random'] = 4 assert d['random'] == 4 assert 'random' in d del d['random'] assert 'random' not in d deep = ['key', 'key1', 'key2'] deep_val = 'mister' d.chain_set(deep, deep_val) assert deep_val == d.chain_get(deep) d.chain_del(deep) try: d.chain_get(deep) except KeyError: pass except Exception: print('failed to throw KeyError') else: print('failed to throw KeyError') assert 'random' not in d d['random'] = 4 dd = RedisDict(namespace='app_name_too') assert len(dd) == 0
import holoviews as hv import numpy as np import pandas as pd import panel as pn import param from bokeh.models import HoverTool, ResetTool, PanTool, WheelZoomTool, ColumnDataSource from bokeh.plotting import figure from holoviews.util.transform import lon_lat_to_easting_northing, easting_northing_to_lon_lat from redis_dict import RedisDict from tqdm import tqdm from loguru import logger as log from constants import CSS_CLASS_CARD from utils import get_price_range, OSM_tile_source from bokeh.models.widgets.tables import HTMLTemplateFormatter, NumberFormatter r_dic = RedisDict(namespace='house-search', host="10.30.40.132") bootstrap = pn.template.BootstrapTemplate(title='Smart House Search') pn.config.sizing_mode = "stretch_width" pn.extension(raw_css=[CSS_CLASS_CARD]) TOOLTIPS = """ <div width="520" style="width:350px;"> <div> <img src="@photo" height="128" alt="@photo" width="128" style="float: left; margin: 0px 15px 15px 0px;" border="2" ></img> </div> <div> <span style="font-size: 12px; font-weight: bold;">@address</span>