def check_availability_and_purchase(customer, event_sku, qty, tier="General"): """Check if there is sufficient inventory before making the purchase""" p = redis.pipeline() try: e_key = keynamehelper.create_key_name("event", event_sku) redis.watch(e_key) available = int(redis.hget(e_key, "available:" + tier)) price = float(redis.hget(e_key, "price:" + tier)) if available >= qty: p.hincrby(e_key, "available:" + tier, -qty) order_id = generate.order_id() purchase = { 'order_id': order_id, 'customer': customer, 'tier': tier, 'qty': qty, 'cost': qty * price, 'event_sku': event_sku, 'ts': long(time.time()) } so_key = keynamehelper.create_key_name("sales_order", order_id) p.hmset(so_key, purchase) p.execute() else: print "Insufficient inventory, have {}, requested {}".format( available, qty) except WatchError: print "Write Conflict check_availability_and_purchase: {}".format( e_key) finally: p.reset() print "Purchase complete!"
def test_expired_res(): """Test function expired reservations""" print "\n==Test 3: Back out reservations when expiration threshold exceeded" # Create events create_events(events) # Create expired reservations for the Event print "== Create ticket holds, expire > 30 sec, return tickets to inventory" event_requested = "320-GHI-921" create_expired_reservation(event_requested) tier = "General" h_key = keynamehelper.create_key_name("ticket_hold", event_requested) e_key = keynamehelper.create_key_name("event", event_requested) while True: expire_reservation(event_requested) outstanding = redis.hmget(h_key, "qty:VPIR6X", "qty:B1BFG7", "qty:UZ1EL0") available = redis.hget(e_key, "available:" + tier) print "{}, Available:{}, Reservations:{}".format( event_requested, available, outstanding) # Break if all items in outstanding list are None if all(v is None for v in outstanding): break else: time.sleep(1)
def print_statistics(stop_event): """Thread that prints current event statsistics.""" from binascii import hexlify sum_key = keynamehelper.create_key_name("sales_summary") print("\n === START") print("{:8} | {:12} | {:3} | Histogram by hour".format( "T/S", "Event", "#"), end=' ') while not stop_event.is_set(): ts = time.strftime("%H:%M:%S") e_key = keynamehelper.create_key_name("event", "*") for event in redis.scan_iter(match=e_key): (_, event_sku) = event.rsplit(":", 1) field_key = keynamehelper.create_field_name( event_sku, "total_tickets_sold") t_tickets = redis.hget(sum_key, field_key) t_tickets = int(t_tickets) if t_tickets != None else 0 tod_hist_key = keynamehelper.create_key_name( "sales_histogram", "time_of_day", event_sku) hist = redis.get(tod_hist_key) if hist != None: hist_vals = [hist[i:i + 2] for i in range(0, len(hist), 2)] print("\n{:8} | {:12} | {:3d} | ".format( ts, event_sku, t_tickets), end=' ') for i in range(0, 24): num = int(hexlify(hist_vals[i].encode( encoding='utf-8')), 16) if i < len(hist_vals) else 0 print("{:02d}/{:03d}".format(i, num), end=' ') time.sleep(1) print("\n === END")
def create_venues(fn="/src/redisu/ru101/data/venues.json"): """Create venues from the flatfile JSON representation""" import json random.seed(94002) f = open(fn) venues = json.load(f) for i in range(len(venues)): v = venues[i] attrs = {'zone': v['zone']} if 'capacity' in v: attrs['capacity'] = v['capacity'] p.hmset(create_key_name("venue", v['venue']), attrs) p.sadd(create_key_name("venues"), v['venue']) for k in range(len(v['events'])): e = create_event(v['events'][k], v['venue'], v['capacity'] if ('capacity' in v) else None, v['geo'] if ('geo' in v) else None, True, True, True) p.sadd(create_key_name("venue", v['venue'], "events"), e['sku']) if 'transit' in v: for k in range(len(v['transit'])): create_transit(v['transit'][k], v['venue'], e['sku'], v['geo'] if ('geo' in v) else None) if 'geo' in v: p.geoadd(create_key_name("geo", "venues"), v['geo']['long'], v['geo']['lat'], v['venue']) p.execute()
def reservation(event_sku, tier, block_name, first_seat, last_seat): """ Reserve the required seats. Create an expiring key (i.e. a latch) to reserve each seat. If that is successful, then an XOR can be executed to update the seat map, without needed a Watch.""" reserved = False p = redis.pipeline() try: for i in range(first_seat, last_seat + 1): # Reserve individual seat, raise exception is already reserved seat_key = keynamehelper.create_key_name("seatres", event_sku, tier, block_name, str(i)) if redis.set(seat_key, True, px=5000, nx=True) != True: raise SeatTaken(i, seat_key) order_id = generate.order_id() required_block = int(math.pow(2, last_seat - first_seat + 1)) - 1 << ( first_seat - 1) vals = ["SET", "u32", 0, required_block] res_key = keynamehelper.create_key_name("seatres", event_sku, tier, block_name, order_id) p.execute_command("BITFIELD", res_key, *vals) p.expire(res_key, 5) block_key = keynamehelper.create_key_name("seatmap", event_sku, tier, block_name) p.bitop("XOR", block_key, block_key, res_key) p.execute() reserved = True except SeatTaken as error: print "Seat Taken/{}".format(error.message) finally: p.reset() return reserved
def reserve(customer, event_sku, qty, tier="General"): """First reserve the inventory and perform a credit authorization. If successful then confirm the inventory deduction or back the deducation out.""" p = redis.pipeline() try: e_key = keynamehelper.create_key_name("event", event_sku) redis.watch(e_key) available = int(redis.hget(e_key, "available:" + tier)) if available >= qty: order_id = generate.order_id() ts = long(time.time()) price = float(redis.hget(e_key, "price:" + tier)) p.hincrby(e_key, "available:" + tier, -qty) p.hincrby(e_key, "held:" + tier, qty) # Create a hash to store the seat hold information hold_key = keynamehelper.create_key_name("ticket_hold", event_sku) p.hsetnx(hold_key, "qty:" + order_id, qty) p.hsetnx(hold_key, "tier:" + order_id, tier) p.hsetnx(hold_key, "ts:" + order_id, ts) p.execute() except WatchError: print "Write Conflict in reserve: {}".format(e_key) finally: p.reset() if creditcard_auth(customer, qty * price): try: purchase = { 'order_id': order_id, 'customer': customer, 'tier': tier, 'qty': qty, 'cost': qty * price, 'event_sku': event_sku, 'ts': long(time.time()) } redis.watch(e_key) # Remove the seat hold, since it is no longer needed p.hdel( hold_key, "qty:" + order_id, ) p.hdel(hold_key, "tier:" + order_id) p.hdel(hold_key, "ts:" + order_id) # Update the Event p.hincrby(e_key, "held:" + tier, -qty) # Post the Sales Order so_key = keynamehelper.create_key_name("sales_order", order_id) p.hmset(so_key, purchase) p.execute() except WatchError: print "Write Conflict in reserve: {}".format(e_key) finally: p.reset() print "Purchase complete!" else: print "Auth failure on order {} for customer {} ${}".format( order_id, customer, price * qty) backout_hold(event_sku, order_id)
def post_purchases(order_id, s_order): """Publish purchases to the queue.""" so_key = keynamehelper.create_key_name("sales_order", order_id) redis.hmset(so_key, s_order) notify_key = keynamehelper.create_key_name("sales_order_notify") redis.publish(notify_key, order_id) notify_key = keynamehelper.create_key_name("sales_order_notify", s_order['event']) redis.publish(notify_key, order_id)
def listener_event_alerter(channel): """Listener for purchases for events other than 'Opening Ceremony'.""" l = redis.pubsub(ignore_subscribe_messages=True) c_key = keynamehelper.create_key_name(channel, "[^Opening]*") l.psubscribe(c_key) for message in l.listen(): order_id = message['data'] so_key = keynamehelper.create_key_name("sales_order", order_id) (event_sku, qty, cost) = redis.hmget(so_key, 'event', 'qty', 'cost') print("Purchase {}: #{} ${}".format(event_sku, qty, cost))
def create_events_hashed_lookups(e_array): """Create hashed lookup for each event""" for i in range(len(e_array)): key = keynamehelper.create_key_name("event", e_array[i]['sku']) redis.set(key, json.dumps(e_array[i])) hfs = [] for key in range(len(__lookup_attrs__)): if __lookup_attrs__[key] in e_array[i]: hfs.append((__lookup_attrs__[key], e_array[i][__lookup_attrs__[key]])) hashed_val = hashlib.sha256(str(hfs)).hexdigest() hfs_key = keynamehelper.create_key_name("hfs", hashed_val) redis.sadd(hfs_key, e_array[i]['sku'])
def create_customers(num): """Generate customer profiles""" fake.seed(94002) for _ in range(num): cust_id = redisu.ru101.common.generate.cust_id() attr = {'customer_name': fake.name(), 'address': fake.address(), 'phone': fake.phone_number()} p.hmset(create_key_name("customer", cust_id), attr) p.sadd(create_key_name("customers"), cust_id) p.execute() customers.append(cust_id)
def create_events_with_lookups(e_array): """Match method 2 - Faceted Search For each attribute & value combination, add the event into a Set""" for i in range(len(e_array)): key = keynamehelper.create_key_name("event", e_array[i]['sku']) redis.set(key, json.dumps(e_array[i])) for k in range(len(__lookup_attrs__)): if __lookup_attrs__[k] in e_array[i]: attr_name = str(e_array[i][__lookup_attrs__[k]]) fs_key = keynamehelper.create_key_name("fs", __lookup_attrs__[k], attr_name) redis.sadd(fs_key, e_array[i]['sku'])
def create_events(event_array, available=None, price=None, tier="General"): """ Create events from the array of passed event details. Provides overrides for number of available tickets, price and ticket tier.""" e_set_key = keynamehelper.create_key_name("events") for event in event_array: # Override the availability & price if provided if available != None: event['available:' + tier] = available if price != None: event['price:' + tier] = price e_key = keynamehelper.create_key_name("event", event['sku']) redis.hmset(e_key, event) redis.sadd(e_set_key, event['sku'])
def create_expired_reservation(event_sku, tier="General"): """Test function to create a set of reservation that will shortly expire""" cur_t = time.time() tickets = {'available:' + tier: 485, 'held:' + tier: 15} holds = {'qty:VPIR6X': 3, 'tier:VPIR6X': tier, 'ts:VPIR6X': int(cur_t - 16), 'qty:B1BFG7': 5, 'tier:B1BFG7': tier, 'ts:B1BFG7': int(cur_t - 22), 'qty:UZ1EL0': 7, 'tier:UZ1EL0': tier, 'ts:UZ1EL0': int(cur_t - 30) } k = keynamehelper.create_key_name("ticket_hold", event_sku) redis.hmset(k, holds) k = keynamehelper.create_key_name("event", event_sku) redis.hmset(k, tickets)
def listener_sales_analytics(channel): """Listener to summarize the sales statistics. Histograms, using BITFIELDs are maintained to show sales by hour.""" l = redis.pubsub(ignore_subscribe_messages=True) c_key = keynamehelper.create_key_name(channel) l.subscribe(c_key) for message in l.listen(): order_id = message['data'] so_key = keynamehelper.create_key_name("sales_order", order_id) (ts, qty, event_sku) = redis.hmget(so_key, 'ts', 'qty', 'event') hour_of_day = int(time.strftime("%H", time.gmtime(int(ts)))) vals = ["INCRBY", "u16", max(hour_of_day * 16, 0), int(qty)] tod_event_hist_key = keynamehelper.create_key_name( "sales_histogram", "time_of_day", event_sku) redis.execute_command("BITFIELD", tod_event_hist_key, *vals)
def create_events(self, event_array, available=None, price=None, tier="General"): """ Create events from an array of event details. Provides overrides for the number of available tickets, price, and ticket tier.""" e_set_key = keynamehelper.create_key_name("events") keys = [] for event in event_array: # Override the availability & price if provided if available != None: event['available:' + tier] = available if price != None: event['price:' + tier] = price e_key = keynamehelper.create_key_name("event", event['sku']) self.redis.hmset(e_key, event) self.redis.sadd(e_set_key, event['sku']) keys.append(e_key) return keys
def listener_ceremony_alerter(channel): """Listener that looks for either 'Opening Ceremony' or 'Closing Ceremony' events only. If then tracks a Lottery content, award a prize for every 5th order for this event only.""" l = redis.pubsub(ignore_subscribe_messages=True) c_key = keynamehelper.create_key_name(channel, "*Ceremony") l.psubscribe(c_key) for message in l.listen(): order_id = message['data'] _, event = message['channel'].rsplit(":", 1) sum_key = keynamehelper.create_key_name("sales_summary") field_key = keynamehelper.create_field_name(event, "total_orders") total_orders = redis.hincrby(sum_key, field_key, 1) if total_orders % 5 == 0: print("===> Winner!!!!! Ceremony Lottery - Order Id: {}"\ .format(order_id))
def find_seats(event_sku, tier, qty): """Find abailable seats""" # Find seat maps import math allocated_seats = [] total_allocated = 0 to_allocate = qty for key in redis.scan_iter(create_key_name("seatmap", event_sku, tier, "*")): available = redis.bitcount(key) if available > 0: vals = ["GET", "u32", 0] new_seat_map = int(redis.execute_command("BITFIELD", key, *vals)[0]) # Take some seats from this block num_taking = max(1, min(available // 2, to_allocate // 2)) pos = list(range(0, 31)) random.shuffle(pos) current_pos = 0 for _ in range(num_taking): if new_seat_map >> pos[current_pos] & 1: new_seat_map -= int(math.pow(2, pos[current_pos])) vals = ["SET", "u32", 0, new_seat_map] redis.execute_command("BITFIELD", key, *vals) block_name = str(key).split(":")[3] allocated_seats.append(block_name + ":" + str(pos[current_pos])) current_pos += 1 total_allocated += 1 to_allocate -= 1 if to_allocate == 0: break if to_allocate == 0: break return {'requested': qty, 'assigned': total_allocated, 'seats': allocated_seats}
def create_faceted_search(obj, key="sku", attrs=search_attrs): """Add keys for faceted search unit""" for k in range(len(attrs)): if search_attrs[k] in obj: fs_key = create_key_name("fs", search_attrs[k], str(obj[search_attrs[k]])) redis.sadd(fs_key, obj[key] if (key in obj) else None)
def test_venue_search(): """Test 1 - geo searches around a venue""" print "\n==Test 1 - geo searches around a venue" create_venue(olympic_stadium) create_venue(nippon_budokan) create_venue(makuhari_messe) create_venue(saitama_super_arena) create_venue(international_stadium) create_venue(isc) print "== Find venues with 5km of 'Tokyo Station'" geo_key = keynamehelper.create_key_name("geo", "venues") print redis.georadius(geo_key, 139.771977, 35.668024, 5, "km", withdist=True) print "== Find venues within 25km of 'Olympic Stadium'" print redis.georadiusbymember(geo_key, "Olympic Stadium", 25, "km", withdist=True)
def test_transit_search(): """Test 3 - geo searched around transit""" print "\n==Test 3 - geo searched around transit" create_event_transit_locations(olympic_stadium) create_event_transit_locations(nippon_budokan) create_event_transit_locations(makuhari_messe) create_event_transit_locations(saitama_super_arena) create_event_transit_locations(international_stadium) create_event_transit_locations(isc) print "== Find venues 5km from 'Tokyo Station' on the 'Keiyo Line'" geo_key = keynamehelper.create_key_name("geo", "transits", "Keiyo Line") print redis.georadius(geo_key, 139.771977, 35.668024, 5, "km", withdist=True) print """== Find the distance between 'Makuhari Messe' and 'Tokyo Tatsumi International Swimming Center' on the 'Keiyo Line'""" print redis.geodist(geo_key, "Makuhari Messe", "Tokyo Tatsumi International Swimming Center", "km") print "== Find venues within 20km of 'Makuhari Messe' on the 'Keiyo Line'" # Note: This only works if the member we are search for is on the # "Keiyo Line". For example, "Olympic Statdium" is not # on the "Keiyo Line" so would return zero results. print redis.georadiusbymember(geo_key, "Makuhari Messe", 20, "km", withdist=True)
def create_customers(self, cust_array): """Create customer keys from an array of customer details""" keys = [] for cust in cust_array: c_key = keynamehelper.create_key_name("customer", cust['id']) self.redis.hmset(c_key, cust) keys.append(c_key) return keys
def match_by_faceting(*keys): """Use SINTER to find the matching elements""" facets = [] for keyval in keys: key, val = keyval fs_key = keynamehelper.create_key_name("fs", key, str(val)) facets.append(fs_key) return redis.sinter(facets)
def create_event_transit_locations(venue): """Create geo entries for transit stops for the passed venue""" p = redis.pipeline() for i in range(len(venue['transit'])): key = keynamehelper.create_key_name("geo", "transits", venue['transit'][i]) p.geoadd(key, venue['geo']['long'], venue['geo']['lat'], venue['venue']) p.execute()
def create_event_locations(venue): """Create geo entry for venues""" p = redis.pipeline() for i in range(len(venue['events'])): event, _ = venue['events'][i] key = keynamehelper.create_key_name("geo", "events", event) p.geoadd(key, venue['geo']['long'], venue['geo']['lat'], venue['venue']) p.execute()
def create_purchase(self, customer, event, quantity): order_id = generate.order_id() purchase_key = keynamehelper.create_key_name("sales_order", order_id) purchase = {'state':'RESERVE', 'order_id': order_id, 'customer_id': customer['id'], 'qty': quantity, 'cost': quantity * float(event['price:General']), 'event_sku': event['sku'], 'ts': int(time.time())} self.redis.hmset(purchase_key, purchase) return purchase_key
def create_hashed_search(obj, key="sku", attrs=search_attrs): """Add keys for hashed search unit""" import hashlib hfs = [] for k in range(len(attrs)): if search_attrs[k] in obj: hfs.append((search_attrs[k], obj[search_attrs[k]])) hfs_k = create_key_name("hfs", hashlib.sha256(str(hfs).encode('utf-8')).hexdigest()) redis.sadd(hfs_k, obj[key] if (key in obj) else None)
def expire_reservation(event_sku, cutoff_time_secs=30): """ Check if any reservation has exceeded the cutoff time. If any have, then backout the reservation and return the inventory back to the pool.""" cutoff_ts = long(time.time() - cutoff_time_secs) e_key = keynamehelper.create_key_name("ticket_hold", event_sku) for field in redis.hscan_iter(e_key, match="ts:*"): if long(field[1]) < cutoff_ts: (_, order_id) = field[0].split(":") backout_hold(event_sku, order_id)
def listener_events_analytics(channel): """Listener to summarize total sales by ticket numbers and order value.""" l = redis.pubsub(ignore_subscribe_messages=True) c_key = keynamehelper.create_key_name(channel) l.subscribe(c_key) p = redis.pipeline() for message in l.listen(): order_id = message['data'] so_key = keynamehelper.create_key_name("sales_order", order_id) (cost, qty, event_sku) = redis.hmget(so_key, 'cost', 'qty', 'event') so_set_key = keynamehelper.create_key_name("sales", event_sku) p.sadd(so_set_key, order_id) sum_key = keynamehelper.create_key_name("sales_summary") p.hincrbyfloat(sum_key, keynamehelper.create_field_name(event_sku, "total_sales"), cost) p.hincrby(sum_key, keynamehelper.create_field_name(event_sku, "total_tickets_sold"), qty) p.execute()
def backout_hold(event_sku, order_id): """Remove the ticket reservation""" p = redis.pipeline() try: hold_key = keynamehelper.create_key_name("ticket_hold", event_sku) e_key = keynamehelper.create_key_name("event", event_sku) redis.watch(e_key) qty = long(redis.hget(hold_key, "qty:" + order_id)) tier = redis.hget(hold_key, "tier:" + order_id) p.hincrby(e_key, "available:" + tier, qty) p.hincrby(e_key, "held:" + tier, -qty) # Remove the hold, since it is no longer needed p.hdel(hold_key, "qty:" + order_id) p.hdel(hold_key, "tier:" + order_id) p.hdel(hold_key, "ts:" + order_id) p.execute() except WatchError: print "Write Conflict in backout_hold: {}".format(e_key) finally: p.reset()
def test_reserved_seats(): """Test function for reserving seats""" print "\n==Test - Reserved Seats" print "== Block of 10 seats, with seat 4 taken" event = "737-DEF-911" seats = 10 create_event(event, 1, seats, "VIP") # Seat 4 (the 8th bit) is already sold. We calc this as # (2^(seats)-1) - bit_number_of_seat # e.g. 1023 - 8 set_seat_map(event, "VIP", "A", int(math.pow(2, seats) - 1 - 8)) print_event_seat_map(event) print "== Request 2 seats, succeeds" seats = find_seat_selection(event, "VIP", 2) print_seat_availabiliy(seats) # Just choose the first found made_reservation = reservation(event, "VIP", seats[0]['block'], seats[0]['available'][0]['first_seat'], seats[0]['available'][0]['last_seat']) print "Made reservation? {}".format(made_reservation) print_event_seat_map(event) # Find space for 5 seats print "== Request 5 seats, succeeds" seats = find_seat_selection(event, "VIP", 5) print_seat_availabiliy(seats) # Just choose the first found made_reservation = reservation(event, "VIP", seats[0]['block'], seats[0]['available'][0]['first_seat'], seats[0]['available'][0]['last_seat']) print "Made reservation? {}".format(made_reservation) print_event_seat_map(event) # Find space for 2 seat, but not enough inventory print "== Request 2 seats, fails" seats = find_seat_selection(event, "VIP", 2) if len(seats) == 0: print "Not enough seats" # Find space for 1 seat print "== Simulate two users trying to get the same seat" seats = find_seat_selection(event, "VIP", 1) # Create a seat reservation (simulating another user), so that the # reservation fails seat_num = str(seats[0]['available'][0]['first_seat']) key = keynamehelper.create_key_name("seatres", event, "VIP", seats[0]['block'], seat_num) redis.set(key, True, px=5000) made_reservation = reservation(event, "VIP", seats[0]['block'], seats[0]['available'][0]['first_seat'], seats[0]['available'][0]['last_seat']) print "Made reservation? {}".format(made_reservation) print_event_seat_map(event)