def check_availability_and_purchase(customer, event_sku, qty, tier="General"):
"""Check if there is sufficient inventory before making the purchase"""
p = redis.pipeline()
try:
e_key = keynamehelper.create_key_name("event", event_sku)
redis.watch(e_key)
available = int(redis.hget(e_key, "available:" + tier))
price = float(redis.hget(e_key, "price:" + tier))
if available >= qty:
p.hincrby(e_key, "available:" + tier, -qty)
order_id = generate.order_id()
purchase = {
'order_id': order_id,
'customer': customer,
'tier': tier,
'qty': qty,
'cost': qty * price,
'event_sku': event_sku,
'ts': long(time.time())
}
so_key = keynamehelper.create_key_name("sales_order", order_id)
p.hmset(so_key, purchase)
p.execute()
else:
print "Insufficient inventory, have {}, requested {}".format(
available, qty)
except WatchError:
print "Write Conflict check_availability_and_purchase: {}".format(
e_key)
finally:
p.reset()
print "Purchase complete!"
def test_expired_res():
"""Test function expired reservations"""
print "\n==Test 3: Back out reservations when expiration threshold exceeded"
# Create events
create_events(events)
# Create expired reservations for the Event
print "== Create ticket holds, expire > 30 sec, return tickets to inventory"
event_requested = "320-GHI-921"
create_expired_reservation(event_requested)
tier = "General"
h_key = keynamehelper.create_key_name("ticket_hold", event_requested)
e_key = keynamehelper.create_key_name("event", event_requested)
while True:
expire_reservation(event_requested)
outstanding = redis.hmget(h_key, "qty:VPIR6X", "qty:B1BFG7",
"qty:UZ1EL0")
available = redis.hget(e_key, "available:" + tier)
print "{}, Available:{}, Reservations:{}".format(
event_requested, available, outstanding)
# Break if all items in outstanding list are None
if all(v is None for v in outstanding):
break
else:
time.sleep(1)
def print_statistics(stop_event):
"""Thread that prints current event statsistics."""
from binascii import hexlify
sum_key = keynamehelper.create_key_name("sales_summary")
print("\n === START")
print("{:8} | {:12} | {:3} | Histogram by hour".format(
"T/S", "Event", "#"),
end=' ')
while not stop_event.is_set():
ts = time.strftime("%H:%M:%S")
e_key = keynamehelper.create_key_name("event", "*")
for event in redis.scan_iter(match=e_key):
(_, event_sku) = event.rsplit(":", 1)
field_key = keynamehelper.create_field_name(
event_sku, "total_tickets_sold")
t_tickets = redis.hget(sum_key, field_key)
t_tickets = int(t_tickets) if t_tickets != None else 0
tod_hist_key = keynamehelper.create_key_name(
"sales_histogram", "time_of_day", event_sku)
hist = redis.get(tod_hist_key)
if hist != None:
hist_vals = [hist[i:i + 2] for i in range(0, len(hist), 2)]
print("\n{:8} | {:12} | {:3d} | ".format(
ts, event_sku, t_tickets),
end=' ')
for i in range(0, 24):
num = int(hexlify(hist_vals[i].encode(
encoding='utf-8')), 16) if i < len(hist_vals) else 0
print("{:02d}/{:03d}".format(i, num), end=' ')
time.sleep(1)
print("\n === END")
def create_venues(fn="/src/redisu/ru101/data/venues.json"):
"""Create venues from the flatfile JSON representation"""
import json
random.seed(94002)
f = open(fn)
venues = json.load(f)
for i in range(len(venues)):
v = venues[i]
attrs = {'zone': v['zone']}
if 'capacity' in v:
attrs['capacity'] = v['capacity']
p.hmset(create_key_name("venue", v['venue']), attrs)
p.sadd(create_key_name("venues"), v['venue'])
for k in range(len(v['events'])):
e = create_event(v['events'][k],
v['venue'],
v['capacity'] if ('capacity' in v) else None,
v['geo'] if ('geo' in v) else None,
True,
True,
True)
p.sadd(create_key_name("venue", v['venue'], "events"), e['sku'])
if 'transit' in v:
for k in range(len(v['transit'])):
create_transit(v['transit'][k],
v['venue'],
e['sku'],
v['geo'] if ('geo' in v) else None)
if 'geo' in v:
p.geoadd(create_key_name("geo", "venues"),
v['geo']['long'], v['geo']['lat'], v['venue'])
p.execute()
def reservation(event_sku, tier, block_name, first_seat, last_seat):
""" Reserve the required seats. Create an expiring key (i.e. a latch) to
reserve each seat. If that is successful, then an XOR can be executed to
update the seat map, without needed a Watch."""
reserved = False
p = redis.pipeline()
try:
for i in range(first_seat, last_seat + 1):
# Reserve individual seat, raise exception is already reserved
seat_key = keynamehelper.create_key_name("seatres", event_sku,
tier, block_name, str(i))
if redis.set(seat_key, True, px=5000, nx=True) != True:
raise SeatTaken(i, seat_key)
order_id = generate.order_id()
required_block = int(math.pow(2, last_seat - first_seat + 1)) - 1 << (
first_seat - 1)
vals = ["SET", "u32", 0, required_block]
res_key = keynamehelper.create_key_name("seatres", event_sku, tier,
block_name, order_id)
p.execute_command("BITFIELD", res_key, *vals)
p.expire(res_key, 5)
block_key = keynamehelper.create_key_name("seatmap", event_sku, tier,
block_name)
p.bitop("XOR", block_key, block_key, res_key)
p.execute()
reserved = True
except SeatTaken as error:
print "Seat Taken/{}".format(error.message)
finally:
p.reset()
return reserved
def reserve(customer, event_sku, qty, tier="General"):
"""First reserve the inventory and perform a credit authorization. If successful
then confirm the inventory deduction or back the deducation out."""
p = redis.pipeline()
try:
e_key = keynamehelper.create_key_name("event", event_sku)
redis.watch(e_key)
available = int(redis.hget(e_key, "available:" + tier))
if available >= qty:
order_id = generate.order_id()
ts = long(time.time())
price = float(redis.hget(e_key, "price:" + tier))
p.hincrby(e_key, "available:" + tier, -qty)
p.hincrby(e_key, "held:" + tier, qty)
# Create a hash to store the seat hold information
hold_key = keynamehelper.create_key_name("ticket_hold", event_sku)
p.hsetnx(hold_key, "qty:" + order_id, qty)
p.hsetnx(hold_key, "tier:" + order_id, tier)
p.hsetnx(hold_key, "ts:" + order_id, ts)
p.execute()
except WatchError:
print "Write Conflict in reserve: {}".format(e_key)
finally:
p.reset()
if creditcard_auth(customer, qty * price):
try:
purchase = {
'order_id': order_id,
'customer': customer,
'tier': tier,
'qty': qty,
'cost': qty * price,
'event_sku': event_sku,
'ts': long(time.time())
}
redis.watch(e_key)
# Remove the seat hold, since it is no longer needed
p.hdel(
hold_key,
"qty:" + order_id,
)
p.hdel(hold_key, "tier:" + order_id)
p.hdel(hold_key, "ts:" + order_id)
# Update the Event
p.hincrby(e_key, "held:" + tier, -qty)
# Post the Sales Order
so_key = keynamehelper.create_key_name("sales_order", order_id)
p.hmset(so_key, purchase)
p.execute()
except WatchError:
print "Write Conflict in reserve: {}".format(e_key)
finally:
p.reset()
print "Purchase complete!"
else:
print "Auth failure on order {} for customer {} ${}".format(
order_id, customer, price * qty)
backout_hold(event_sku, order_id)
def post_purchases(order_id, s_order):
"""Publish purchases to the queue."""
so_key = keynamehelper.create_key_name("sales_order", order_id)
redis.hmset(so_key, s_order)
notify_key = keynamehelper.create_key_name("sales_order_notify")
redis.publish(notify_key, order_id)
notify_key = keynamehelper.create_key_name("sales_order_notify",
s_order['event'])
redis.publish(notify_key, order_id)
def listener_event_alerter(channel):
"""Listener for purchases for events other than 'Opening Ceremony'."""
l = redis.pubsub(ignore_subscribe_messages=True)
c_key = keynamehelper.create_key_name(channel, "[^Opening]*")
l.psubscribe(c_key)
for message in l.listen():
order_id = message['data']
so_key = keynamehelper.create_key_name("sales_order", order_id)
(event_sku, qty, cost) = redis.hmget(so_key, 'event', 'qty', 'cost')
print("Purchase {}: #{} ${}".format(event_sku, qty, cost))
def create_events_hashed_lookups(e_array):
"""Create hashed lookup for each event"""
for i in range(len(e_array)):
key = keynamehelper.create_key_name("event", e_array[i]['sku'])
redis.set(key, json.dumps(e_array[i]))
hfs = []
for key in range(len(__lookup_attrs__)):
if __lookup_attrs__[key] in e_array[i]:
hfs.append((__lookup_attrs__[key], e_array[i][__lookup_attrs__[key]]))
hashed_val = hashlib.sha256(str(hfs)).hexdigest()
hfs_key = keynamehelper.create_key_name("hfs", hashed_val)
redis.sadd(hfs_key, e_array[i]['sku'])
def create_customers(num):
"""Generate customer profiles"""
fake.seed(94002)
for _ in range(num):
cust_id = redisu.ru101.common.generate.cust_id()
attr = {'customer_name': fake.name(),
'address': fake.address(),
'phone': fake.phone_number()}
p.hmset(create_key_name("customer", cust_id), attr)
p.sadd(create_key_name("customers"), cust_id)
p.execute()
customers.append(cust_id)
def create_events_with_lookups(e_array):
"""Match method 2 - Faceted Search
For each attribute & value combination, add the event into a Set"""
for i in range(len(e_array)):
key = keynamehelper.create_key_name("event", e_array[i]['sku'])
redis.set(key, json.dumps(e_array[i]))
for k in range(len(__lookup_attrs__)):
if __lookup_attrs__[k] in e_array[i]:
attr_name = str(e_array[i][__lookup_attrs__[k]])
fs_key = keynamehelper.create_key_name("fs",
__lookup_attrs__[k],
attr_name)
redis.sadd(fs_key, e_array[i]['sku'])
def create_events(event_array, available=None, price=None, tier="General"):
""" Create events from the array of passed event details. Provides overrides
for number of available tickets, price and ticket tier."""
e_set_key = keynamehelper.create_key_name("events")
for event in event_array:
# Override the availability & price if provided
if available != None:
event['available:' + tier] = available
if price != None:
event['price:' + tier] = price
e_key = keynamehelper.create_key_name("event", event['sku'])
redis.hmset(e_key, event)
redis.sadd(e_set_key, event['sku'])
def create_expired_reservation(event_sku, tier="General"):
"""Test function to create a set of reservation that will shortly expire"""
cur_t = time.time()
tickets = {'available:' + tier: 485,
'held:' + tier: 15}
holds = {'qty:VPIR6X': 3, 'tier:VPIR6X': tier, 'ts:VPIR6X': int(cur_t - 16),
'qty:B1BFG7': 5, 'tier:B1BFG7': tier, 'ts:B1BFG7': int(cur_t - 22),
'qty:UZ1EL0': 7, 'tier:UZ1EL0': tier, 'ts:UZ1EL0': int(cur_t - 30)
}
k = keynamehelper.create_key_name("ticket_hold", event_sku)
redis.hmset(k, holds)
k = keynamehelper.create_key_name("event", event_sku)
redis.hmset(k, tickets)
def listener_sales_analytics(channel):
"""Listener to summarize the sales statistics. Histograms, using
BITFIELDs are maintained to show sales by hour."""
l = redis.pubsub(ignore_subscribe_messages=True)
c_key = keynamehelper.create_key_name(channel)
l.subscribe(c_key)
for message in l.listen():
order_id = message['data']
so_key = keynamehelper.create_key_name("sales_order", order_id)
(ts, qty, event_sku) = redis.hmget(so_key, 'ts', 'qty', 'event')
hour_of_day = int(time.strftime("%H", time.gmtime(int(ts))))
vals = ["INCRBY", "u16", max(hour_of_day * 16, 0), int(qty)]
tod_event_hist_key = keynamehelper.create_key_name(
"sales_histogram", "time_of_day", event_sku)
redis.execute_command("BITFIELD", tod_event_hist_key, *vals)
def create_events(self, event_array, available=None, price=None, tier="General"):
""" Create events from an array of event details. Provides overrides
for the number of available tickets, price, and ticket tier."""
e_set_key = keynamehelper.create_key_name("events")
keys = []
for event in event_array:
# Override the availability & price if provided
if available != None:
event['available:' + tier] = available
if price != None:
event['price:' + tier] = price
e_key = keynamehelper.create_key_name("event", event['sku'])
self.redis.hmset(e_key, event)
self.redis.sadd(e_set_key, event['sku'])
keys.append(e_key)
return keys
def listener_ceremony_alerter(channel):
"""Listener that looks for either 'Opening Ceremony' or 'Closing Ceremony'
events only. If then tracks a Lottery content, award a prize for every 5th
order for this event only."""
l = redis.pubsub(ignore_subscribe_messages=True)
c_key = keynamehelper.create_key_name(channel, "*Ceremony")
l.psubscribe(c_key)
for message in l.listen():
order_id = message['data']
_, event = message['channel'].rsplit(":", 1)
sum_key = keynamehelper.create_key_name("sales_summary")
field_key = keynamehelper.create_field_name(event, "total_orders")
total_orders = redis.hincrby(sum_key, field_key, 1)
if total_orders % 5 == 0:
print("===> Winner!!!!! Ceremony Lottery - Order Id: {}"\
.format(order_id))
def find_seats(event_sku, tier, qty):
"""Find abailable seats"""
# Find seat maps
import math
allocated_seats = []
total_allocated = 0
to_allocate = qty
for key in redis.scan_iter(create_key_name("seatmap", event_sku, tier, "*")):
available = redis.bitcount(key)
if available > 0:
vals = ["GET", "u32", 0]
new_seat_map = int(redis.execute_command("BITFIELD", key, *vals)[0])
# Take some seats from this block
num_taking = max(1, min(available // 2, to_allocate // 2))
pos = list(range(0, 31))
random.shuffle(pos)
current_pos = 0
for _ in range(num_taking):
if new_seat_map >> pos[current_pos] & 1:
new_seat_map -= int(math.pow(2, pos[current_pos]))
vals = ["SET", "u32", 0, new_seat_map]
redis.execute_command("BITFIELD", key, *vals)
block_name = str(key).split(":")[3]
allocated_seats.append(block_name + ":" + str(pos[current_pos]))
current_pos += 1
total_allocated += 1
to_allocate -= 1
if to_allocate == 0:
break
if to_allocate == 0:
break
return {'requested': qty, 'assigned': total_allocated,
'seats': allocated_seats}
def create_faceted_search(obj, key="sku", attrs=search_attrs):
"""Add keys for faceted search unit"""
for k in range(len(attrs)):
if search_attrs[k] in obj:
fs_key = create_key_name("fs", search_attrs[k],
str(obj[search_attrs[k]]))
redis.sadd(fs_key, obj[key] if (key in obj) else None)
def test_venue_search():
"""Test 1 - geo searches around a venue"""
print "\n==Test 1 - geo searches around a venue"
create_venue(olympic_stadium)
create_venue(nippon_budokan)
create_venue(makuhari_messe)
create_venue(saitama_super_arena)
create_venue(international_stadium)
create_venue(isc)
print "== Find venues with 5km of 'Tokyo Station'"
geo_key = keynamehelper.create_key_name("geo", "venues")
print redis.georadius(geo_key,
139.771977,
35.668024,
5,
"km",
withdist=True)
print "== Find venues within 25km of 'Olympic Stadium'"
print redis.georadiusbymember(geo_key,
"Olympic Stadium",
25,
"km",
withdist=True)
def test_transit_search():
"""Test 3 - geo searched around transit"""
print "\n==Test 3 - geo searched around transit"
create_event_transit_locations(olympic_stadium)
create_event_transit_locations(nippon_budokan)
create_event_transit_locations(makuhari_messe)
create_event_transit_locations(saitama_super_arena)
create_event_transit_locations(international_stadium)
create_event_transit_locations(isc)
print "== Find venues 5km from 'Tokyo Station' on the 'Keiyo Line'"
geo_key = keynamehelper.create_key_name("geo", "transits", "Keiyo Line")
print redis.georadius(geo_key,
139.771977,
35.668024,
5,
"km",
withdist=True)
print """== Find the distance between 'Makuhari Messe' and 'Tokyo Tatsumi
International Swimming Center' on the 'Keiyo Line'"""
print redis.geodist(geo_key, "Makuhari Messe",
"Tokyo Tatsumi International Swimming Center", "km")
print "== Find venues within 20km of 'Makuhari Messe' on the 'Keiyo Line'"
# Note: This only works if the member we are search for is on the
# "Keiyo Line". For example, "Olympic Statdium" is not
# on the "Keiyo Line" so would return zero results.
print redis.georadiusbymember(geo_key,
"Makuhari Messe",
20,
"km",
withdist=True)
def create_customers(self, cust_array):
"""Create customer keys from an array of customer details"""
keys = []
for cust in cust_array:
c_key = keynamehelper.create_key_name("customer", cust['id'])
self.redis.hmset(c_key, cust)
keys.append(c_key)
return keys
def match_by_faceting(*keys):
"""Use SINTER to find the matching elements"""
facets = []
for keyval in keys:
key, val = keyval
fs_key = keynamehelper.create_key_name("fs", key, str(val))
facets.append(fs_key)
return redis.sinter(facets)
def create_event_transit_locations(venue):
"""Create geo entries for transit stops for the passed venue"""
p = redis.pipeline()
for i in range(len(venue['transit'])):
key = keynamehelper.create_key_name("geo", "transits",
venue['transit'][i])
p.geoadd(key, venue['geo']['long'], venue['geo']['lat'],
venue['venue'])
p.execute()
def create_event_locations(venue):
"""Create geo entry for venues"""
p = redis.pipeline()
for i in range(len(venue['events'])):
event, _ = venue['events'][i]
key = keynamehelper.create_key_name("geo", "events", event)
p.geoadd(key, venue['geo']['long'], venue['geo']['lat'],
venue['venue'])
p.execute()
def create_purchase(self, customer, event, quantity):
order_id = generate.order_id()
purchase_key = keynamehelper.create_key_name("sales_order", order_id)
purchase = {'state':'RESERVE', 'order_id': order_id,
'customer_id': customer['id'], 'qty': quantity,
'cost': quantity * float(event['price:General']),
'event_sku': event['sku'], 'ts': int(time.time())}
self.redis.hmset(purchase_key, purchase)
return purchase_key
def create_hashed_search(obj, key="sku", attrs=search_attrs):
"""Add keys for hashed search unit"""
import hashlib
hfs = []
for k in range(len(attrs)):
if search_attrs[k] in obj:
hfs.append((search_attrs[k], obj[search_attrs[k]]))
hfs_k = create_key_name("hfs", hashlib.sha256(str(hfs).encode('utf-8')).hexdigest())
redis.sadd(hfs_k, obj[key] if (key in obj) else None)
def expire_reservation(event_sku, cutoff_time_secs=30):
""" Check if any reservation has exceeded the cutoff time. If any have, then
backout the reservation and return the inventory back to the pool."""
cutoff_ts = long(time.time() - cutoff_time_secs)
e_key = keynamehelper.create_key_name("ticket_hold", event_sku)
for field in redis.hscan_iter(e_key, match="ts:*"):
if long(field[1]) < cutoff_ts:
(_, order_id) = field[0].split(":")
backout_hold(event_sku, order_id)
def listener_events_analytics(channel):
"""Listener to summarize total sales by ticket numbers and order value."""
l = redis.pubsub(ignore_subscribe_messages=True)
c_key = keynamehelper.create_key_name(channel)
l.subscribe(c_key)
p = redis.pipeline()
for message in l.listen():
order_id = message['data']
so_key = keynamehelper.create_key_name("sales_order", order_id)
(cost, qty, event_sku) = redis.hmget(so_key, 'cost', 'qty', 'event')
so_set_key = keynamehelper.create_key_name("sales", event_sku)
p.sadd(so_set_key, order_id)
sum_key = keynamehelper.create_key_name("sales_summary")
p.hincrbyfloat(sum_key,
keynamehelper.create_field_name(event_sku, "total_sales"),
cost)
p.hincrby(sum_key,
keynamehelper.create_field_name(event_sku, "total_tickets_sold"),
qty)
p.execute()
def backout_hold(event_sku, order_id):
"""Remove the ticket reservation"""
p = redis.pipeline()
try:
hold_key = keynamehelper.create_key_name("ticket_hold", event_sku)
e_key = keynamehelper.create_key_name("event", event_sku)
redis.watch(e_key)
qty = long(redis.hget(hold_key, "qty:" + order_id))
tier = redis.hget(hold_key, "tier:" + order_id)
p.hincrby(e_key, "available:" + tier, qty)
p.hincrby(e_key, "held:" + tier, -qty)
# Remove the hold, since it is no longer needed
p.hdel(hold_key, "qty:" + order_id)
p.hdel(hold_key, "tier:" + order_id)
p.hdel(hold_key, "ts:" + order_id)
p.execute()
except WatchError:
print "Write Conflict in backout_hold: {}".format(e_key)
finally:
p.reset()
def test_reserved_seats():
"""Test function for reserving seats"""
print "\n==Test - Reserved Seats"
print "== Block of 10 seats, with seat 4 taken"
event = "737-DEF-911"
seats = 10
create_event(event, 1, seats, "VIP")
# Seat 4 (the 8th bit) is already sold. We calc this as
# (2^(seats)-1) - bit_number_of_seat
# e.g. 1023 - 8
set_seat_map(event, "VIP", "A", int(math.pow(2, seats) - 1 - 8))
print_event_seat_map(event)
print "== Request 2 seats, succeeds"
seats = find_seat_selection(event, "VIP", 2)
print_seat_availabiliy(seats)
# Just choose the first found
made_reservation = reservation(event, "VIP", seats[0]['block'],
seats[0]['available'][0]['first_seat'],
seats[0]['available'][0]['last_seat'])
print "Made reservation? {}".format(made_reservation)
print_event_seat_map(event)
# Find space for 5 seats
print "== Request 5 seats, succeeds"
seats = find_seat_selection(event, "VIP", 5)
print_seat_availabiliy(seats)
# Just choose the first found
made_reservation = reservation(event, "VIP", seats[0]['block'],
seats[0]['available'][0]['first_seat'],
seats[0]['available'][0]['last_seat'])
print "Made reservation? {}".format(made_reservation)
print_event_seat_map(event)
# Find space for 2 seat, but not enough inventory
print "== Request 2 seats, fails"
seats = find_seat_selection(event, "VIP", 2)
if len(seats) == 0:
print "Not enough seats"
# Find space for 1 seat
print "== Simulate two users trying to get the same seat"
seats = find_seat_selection(event, "VIP", 1)
# Create a seat reservation (simulating another user), so that the
# reservation fails
seat_num = str(seats[0]['available'][0]['first_seat'])
key = keynamehelper.create_key_name("seatres", event, "VIP",
seats[0]['block'], seat_num)
redis.set(key, True, px=5000)
made_reservation = reservation(event, "VIP", seats[0]['block'],
seats[0]['available'][0]['first_seat'],
seats[0]['available'][0]['last_seat'])
print "Made reservation? {}".format(made_reservation)
print_event_seat_map(event)