def merge_clock(rating, clock, key):
global client
# make sure the clock is a VectorClock object first
if not isinstance(clock, VectorClock) and isinstance(clock, dict):
clock = VectorClock.fromDict(clock)
#if not isinstance(clock, VectorClock) and not isinstance(clock,dict):
# pdb.set_trace()
clockDict = clock.asDict()
# lets get the hash from redis for this tea-x
teaHash = client.hgetall(key)
#flag so we know whether to just add the new rating or not
isClientExist = False
for clockJsonString, redisRating in teaHash.iteritems():
redisClockDict = json.loads(clockJsonString)
redisClock = VectorClock.fromDict(redisClockDict)
# Check if the clock is inside the redis
if clock >= redisClock or clock < redisClock:
isClientExist = True # well, looks like we won't be creating a new one
# returns either [redisclock], [clock], or [redisClock, clock]
# which means stale, recent, incomparable respectively
vcl = coalesce(redisClock, clock)
# lets cache the comparisons
redisClockIncluded = redisClock in vcl
clockIncluded = clock in vcl
# the incomparable case, include both!
if redisClockIncluded and clockIncluded:
# The redis (clock,rating) is already added so we just add
# the new one to redis
client.hset(key, json.dumps(clockDict), rating)
# check if we can delete the old clock if a client in the new clock
# is newer than an old client's clock time.
for cli in clockDict:
if redisClockDict[cli] < clockDict[cli]:
client.hdel(key, json.dumps(redisClockDict))
# the more recent case, replace the old one with the new one
if not redisClockIncluded and clockIncluded:
client.hdel(key, json.dumps(redisClockDict))
client.hset(key, json.dumps(clockDict), rating)
# Client never rated yet since we didn't find it in the hash so add it
if isClientExist == False:
client.hset(key, json.dumps(clockDict), rating)
# calls method that returns the result as {rating, choices, clock}
final_rating_result = get_final_rating_result(key)
return final_rating_result
def get(id, ec=False, port=lb_base):
""" Get a value.
By default, this will issue a strongly consistent read to the
load balancer. Setting ec=True will request an eventually
consistent read. Setting port to the port of a DB instance
does a direct get to that instance, bypassing the load balancer.
"""
headers = { 'Accept': 'application/json' }
url = endpoint(id, port)
try:
if ec:
response = requests.get(url, headers=headers, params={'consistency': 'weak'})
else:
response = requests.get(url, headers=headers)
except Exception as e:
raise Exception("Invalid request: url %s, exception %s" % (url, e))
try:
data = response.json()
except:
raise Exception('Unexpected response: %s HTTP %d %s' % (url, response.status_code, response.text))
try:
rating = float(data['rating'])
except:
rating = data['rating']
choices = data['choices']
#TODO: Handle return of malformed vector clock
clocks = data['clocks']
return rating, choices, [VectorClock.fromDict(vcstr) for vcstr in clocks]
def put_rating(entity):
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
# Parse the request
data = json.load(request.body)
setrating = data.get('rating')
setclock = VectorClock.fromDict(data.get('clock'))
key = '/rating/'+entity
#key = '/rating-and-clock/'+entity
#sync_with_neighbour_queue(key)
merge_with_db(setrating, setclock, key)
sync_with_neighbour_queue(key)
# lets grab the results of our work!
result = get_final_rating_result(key)
#final_rating_key = '/rating/'+entity
#client.hset(final_rating_key, 'rating', result["rating"])
# Return rating
return {
"rating": result["rating"]
}
def get_rating(entity):
print "Called GET on ENTITY", entity
key = '/rating/'+entity
hashed_entity = hash(entity)
db_index = hashed_entity % ndb
print "entity primary is", db_index
#Calculate rating
clocks = {}
choices = 0
#print "TABLE", table
print "doing GOSSIP for read:", clocks
gossip_protocol()
table = client.hgetall(key)
clocks = table.keys()
choices = table.values()
print "CLOCKS", clocks
print "CHOICES:", choices
print "TYPE OF CLOCKS", type(clocks)
total = 0
for i in range (len(choices)):
choices[i] = float(choices[i])
total = total + choices[i]
clocksArr = []
for clock in clocks:
print "CLOCK", type(clock)
print "CLOCK", clock
clock = VectorClock.fromDict(ast.literal_eval(clock))
clocksArr.append(clock.asDict())
print "TYPE OF CLOCK", type(clock.asDict())
print "CLOCKSARR", clocksArr
checkDigestList()
if (total > 0):
finalrating = float(total/len(choices))
return {
"rating": finalrating,
"choices": choices,
"clocks": clocksArr
}
elif (len(clocksArr) > 0):
return { "rating": 0,
"choices": choices,
"clocks": clocksArr}
else:
return { "rating": 0,
"choices": [],
"clocks": []}
def get(self, channel):
resp = requests.get('http://localhost:'+str(self.port)+'/q/'+channel,
headers={'content-type': 'application/json'})
jresp = resp.json()
if len(jresp) > 0:
for k in jresp:
if (isinstance(jresp[k],dict) and
jresp[k].keys() == [CLOCK_CODE]):
jresp[k] = [VectorClock.fromDict(dc) for dc in jresp[k][CLOCK_CODE]]
return jresp
else:
return None
def process_request_data(entity, rating, newclocks):
print "processing request data"
# Weave the new rating into the current rating list
ratingkey = ""
if "rating" not in entity:
ratingkey = '/rating/'+entity
else:
ratingkey = entity
spectator = True
newClockList = []
table = client.hgetall(ratingkey)
clocks = table.keys()
choices = table.values()
print "table", table
print "clocks", clocks
print "choices", choices
print "newclocks", newclocks
print "type of newclocks", type(newclocks)
if (len(clocks) == 0):
newClockList.append(newclocks)
print "appending new clock to our clock list because our clock list is empty. NewClockList:", newClockList
print "type of newclocks", type(newclocks)
for entry in clocks:
entry = VectorClock.fromDict(ast.literal_eval(entry))
if (entry < newclocks):
print "Our current clock is older!", entry, newclocks
spectator = False
print "STORING IN REDIS DATABASE", newclocks.asDict(), rating
client.hset(ratingkey, newclocks.asDict(), rating)
client.hdel(ratingkey, entry.asDict())
newClockList.append(newclocks)
elif (newclocks <= entry):
spectator = False
if(spectator):
print "spectator", spectator
for clock in newClockList:
print type(clock)
print "STORING IN REDIS DATABASE", ratingkey, clock.asDict(), rating
client.hset(ratingkey, clock.asDict(), rating)
# Return the new rating for the entity
return rating, newClockList
def put_rating(entity):
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
# Parse the request
data = json.load(request.body)
rating = data.get('rating')
clock = VectorClock.fromDict(data.get('clock'))
# Basic sanity checks on the rating
if isinstance(rating, int): rating = float(rating)
if not isinstance(rating, float): return abort(400)
# YOUR CODE HERE
# HASH THE ENTITY TO DETERMINE ITS SHARD
hashed_entity = hash(entity)
# find out which database process this entity hashes to
db_index = hashed_entity % ndb
print "ENTITY has PRIMARY DB as", db_index, entity
# change the dbBasePort to be the port on which the hashed database is talking on
dbBasePort = 3000
dbBasePort = dbBasePort + db_index
# PUT THE PORT FOR THE CORRECT SHARD IN url below
url = 'http://localhost:'+str(dbBasePort)+'/rating/'+entity
print "URL", url
# RESUME BOILERPLATE CODE...
# Update the rating
res = requests.put(url,
data=json.dumps({'rating': rating,
'clock': clock.asDict()}),
headers={'content-type': 'application/json'}
)
# Return the new rating for the entity
return {
"rating": res.json()['rating']
}
def get_from_redis(tea_name):
try:
data = eval(client.get("tea:%s:json" % tea_name))
except:
#print "*** Error [get from redis] no key found for the tea:" + tea_name
return None,None,None
try:
rating = data["rating"]
except:
rating = data["rating"]
choices = data["choices"]
clocks = data["clocks"] #vc.clock, ex) {'c1': 10, 'c0': 7}
return rating, choices, VectorClock.fromDict(clocks)
def put_rating(entity):
print "Called PUT on ENTITY", entity
hashed_entity = hash(entity)
db_index = hashed_entity % ndb
print "entity primary is", db_index
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
# Parse the request
data = json.load(request.body)
setrating = data.get('rating')
setclock = VectorClock.fromDict(data.get('clock'))
key = '/rating/'+ entity
finalrating = 0
newClockList = []
print "ENTITY", entity
print "setrating", setrating
print "setclock", setclock
print "write rating to REDIS Database:", entity, setrating, setclock
(finalrating, newClockList) = process_request_data(entity, setrating, setclock)
print "newClockList:", newClockList
# record change in digest list
global digest_list
print "record change in digest_list:", id, key, finalrating, setclock
for clock in newClockList:
digest_list.append({"id": id, "key": key, "rating": finalrating, "clock": clock})
gossip_protocol()
checkDigestList()
# Return rating
return {
"rating": finalrating
}
def put_rating(entity):
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
data = json.load(request.body)
if DEBUG:
print "[put_rating]", data
recieved_rating = data.get('rating')
recieved_vc = VectorClock.fromDict(data.get('clocks'))
# Basic sanity checks on the rating
if isinstance(recieved_rating, int):recieved_rating = float(recieved_rating)
if not isinstance(recieved_rating, float): return abort(400)
# Weave the new rating into the current rating list
key = '/rating/'+entity
tea_name = entity
# COMPUTE THE MEAN, finalrating after converge existing and recieving value
finalrating, choices, new_vc_list = vector_converge(tea_name,recieved_rating,recieved_vc)
# SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY
if choices!=None:
put_to_redis(tea_name, finalrating,choices,new_vc_list) #store new score
# YOUR CODE HERE
# MERGE WITH CURRENT VALUES FOR THIS KEY
# REPLACE FOLLOWING WITH CORRECT FINAL RATING
# finalrating = 0.0
# SAVE NEW VALUES
# GOSSIP
# Return rating
return {
"rating": finalrating
}
def get(id):
headers = { 'Accept': 'application/json' }
url = endpoint+'/rating/'+id
try:
request = requests.get(url, headers=headers)
data = request.json()
except:
raise Exception('Invalid request: %s HTTP %d %s' % (url, request.status_code, request.text))
try:
rating = float(data['rating'])
except:
rating = data['rating']
choices = json.loads(data['choices'])
#TODO: Handle return of malformed vector clock
clocks = json.load(StringIO.StringIO(data['clocks']))
return rating, choices, [VectorClock.fromDict(vcstr) for vcstr in clocks]
def put_rating(entity):
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
# Parse the request
data = json.load(request.body)
rating = data.get('rating')
clock = VectorClock.fromDict(data.get('clock'))
query_param_dict = parse_qs(urlparse(request.url).query, keep_blank_values=True)
# Basic sanity checks on the rating
if isinstance(rating, int): rating = float(rating)
if not isinstance(rating, float): return abort(400)
# WE ARE NOT SUPPOSE TO HAVE EVENTUAL CONSISTENT WRITES. What does that even mean...
#if 'consistency' in query_param_dict.keys() and query_param_dict['consistency'] == 'weak':
# shard_number = get_shard_number(entity)
#else:
# shard_number = get_shard_number(entity, consistent=True)
dbBasePort = get_shard_number(entity, consistent=True)
url = 'http://localhost:'+str(dbBasePort)+'/rating/'+entity
res = requests.put(url,
data=json.dumps({'rating': rating,
'clock': clock.asDict()}),
headers={'content-type': 'application/json'})
# Return the new rating for the entity
return {
"rating": res.json()['rating']
}
def gossip_protocol():
# first check the channel to see if there is anything in there
isItemInChannel = True
newClockList = []
global queue
global id
global digest_list
while (isItemInChannel):
#TODO: remember do this only if ndb > 1
msg = []
if (id == 0):
msg = queue.get(str(ndb - 1))
if (not msg):
isItemInChannel = False
else:
msg = queue.get(str(id - 1))
if (not msg):
isItemInChannel = False
print "isItemInChannel", isItemInChannel
if (isItemInChannel):
print "MSG", msg
if (id != msg['id']):
ratingValue = msg['rating']
key = msg['key']
clock = msg['clock']
# turn the clock retrieved off channel into a vector clock type
clock = VectorClock.fromDict(clock)
# TODO should I be grabbing the clock from the msg and storing that in REDIS
print "writing this key retrieved off the channel to REDIS", key, ratingValue, clock
(ratingValue, newClockList) = process_request_data(key, ratingValue, clock)
print "append these values to this instance's digest_list, for later gossip to its neighbour:", ratingValue, clock
digest_list.append({"id": id, "key": key, "rating": ratingValue, "clock": clock})
def put_rating(entity):
# Check to make sure JSON is ok
type = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not type: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', type)
# Read the data sent from the client
data = json.load(request.body)
recieved_rating = data.get('rating')
recieved_vc = VectorClock.fromDict(data.get('clocks'))
# Basic sanity checks on the rating
if isinstance(recieved_rating, int):recieved_rating = float(recieved_rating)
if not isinstance(recieved_rating, float): return abort(400)
# Weave the new rating into the current rating list
key = '/rating/'+entity
tea_name = entity
# COMPUTE THE MEAN, finalrating after converge existing and recieving value
finalrating, choices, new_vc = vector_converge(tea_name,recieved_rating,recieved_vc)
# SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY
if choices!=None:
put_to_redis(tea_name, finalrating,choices,new_vc) #store new score
# Return the new rating for the entity
return {
"rating": finalrating
}
def put_rating(entity):
# Check to make sure JSON is ok
mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not mimetype: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', mimetype)
# Parse the request
data = json.load(request.body)
rating = data.get('rating')
clock = VectorClock.fromDict(data.get('clock'))
# Basic sanity checks on the rating
if isinstance(rating, int): rating = float(rating)
if not isinstance(rating, float): return abort(400)
# YOUR CODE HERE
# HASH THE ENTITY TO DETERMINE ITS SHARD
# PUT THE PORT FOR THE CORRECT SHARD IN url below
url = 'http://localhost:'+str(dbBasePort+hashEntity(entity, ndb))+'/rating/'+entity
# RESUME BOILERPLATE CODE...
# Update the rating
res = requests.put(url,
data=json.dumps({'rating': rating,
'clock': clock.asDict()}),
headers={'content-type': 'application/json'})
# Return the new rating for the entity
return {
"rating": res.json()['rating']
}
def put_rating(entity):
# Check to make sure JSON is ok
type = mimeparse.best_match(['application/json'], request.headers.get('Accept'))
if not type: return abort(406)
# Check to make sure the data we're getting is JSON
if request.headers.get('Content-Type') != 'application/json': return abort(415)
response.headers.append('Content-Type', type)
# Read the data sent from the client
data = json.load(request.body)
setrating = data.get('rating')
setclock = VectorClock.fromDict(data.get('clocks'))
# Basic sanity checks on the rating
if isinstance(setrating, int): setrating = float(setrating)
if not isinstance(setrating, float): return abort(400)
# Weave the new rating into the current rating list
key = '/rating/'+entity
# YOUR CODE GOES HERE
# REPLACE THE FOLLOWING LINE
# HINT: CONSIDER USING THE HASH DATA TYPE IN REDIS (HGET/HSET/...)
old_rating = client.hget(key, 'rating')
# if rating does not exist, add it. Otherwise..
# SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY
# COMPUTE THE MEAN, finalrating
if not old_rating:
client.hset(key, 'rating', setrating)
client.hset(key, 'choices', [setrating])
client.hset(key, 'clocks', jsonify_vcl([setclock]))
finalrating = setrating
else:
finalrating = old_rating
choices = eval(client.hget(key, 'choices'))
vcl = eval(client.hget(key, 'clocks'))
new_vcl = []
new_choices = []
greaterThanAlreadyFound = False
needToUpdateDB = True
for i in range(0, len(vcl)):
old_clock = VectorClock.fromDict(vcl[i])
# if the received clock is older, nothing needs updating
if setclock <= old_clock:
needToUpdateDB = False
break
else:
# if the received clock is newer, make changes accordingly
if setclock > old_clock:
# If we have not found an older clock and replaced it with the
# new one previously, put this new clock in. Otherwise, ignore.
if not greaterThanAlreadyFound:
greaterThanAlreadyFound = True
new_vcl.append(setclock)
new_choices.append(setrating)
# incomparable
else:
new_vcl.append(old_clock)
new_choices.append(choices[i])
# Update DB only if the received clock is not older than or the same as any of the
# existing clocks
if needToUpdateDB:
# if the received clock is not newer than any of the existing clocks, it's
# incomparable
if not greaterThanAlreadyFound:
new_vcl.append(setclock)
new_choices.append(setrating)
# calculate the new rating
ratingSum = 0.0
for choice in new_choices:
ratingSum+=choice
finalrating = ratingSum/len(new_choices)
# update DB
client.hset(key, 'rating', finalrating)
client.hset(key, 'choices', new_choices)
client.hset(key, 'clocks', jsonify_vcl(new_vcl))
# Return the new rating for the entity
return {
"rating": finalrating
}
def merge_dict(vc_list):
json_dict = "{%s}" % ", ".join(["%s" % vc.asItemString()
for vc in vc_list ])
return VectorClock.fromDict(eval(json_dict))
def convert2vc_list(json_clocks_list): return [VectorClock.fromDict(clocks) for clocks in eval(json_clocks_list)]
