def merge_clock(rating, clock, key): global client # make sure the clock is a VectorClock object first if not isinstance(clock, VectorClock) and isinstance(clock, dict): clock = VectorClock.fromDict(clock) #if not isinstance(clock, VectorClock) and not isinstance(clock,dict): # pdb.set_trace() clockDict = clock.asDict() # lets get the hash from redis for this tea-x teaHash = client.hgetall(key) #flag so we know whether to just add the new rating or not isClientExist = False for clockJsonString, redisRating in teaHash.iteritems(): redisClockDict = json.loads(clockJsonString) redisClock = VectorClock.fromDict(redisClockDict) # Check if the clock is inside the redis if clock >= redisClock or clock < redisClock: isClientExist = True # well, looks like we won't be creating a new one # returns either [redisclock], [clock], or [redisClock, clock] # which means stale, recent, incomparable respectively vcl = coalesce(redisClock, clock) # lets cache the comparisons redisClockIncluded = redisClock in vcl clockIncluded = clock in vcl # the incomparable case, include both! if redisClockIncluded and clockIncluded: # The redis (clock,rating) is already added so we just add # the new one to redis client.hset(key, json.dumps(clockDict), rating) # check if we can delete the old clock if a client in the new clock # is newer than an old client's clock time. for cli in clockDict: if redisClockDict[cli] < clockDict[cli]: client.hdel(key, json.dumps(redisClockDict)) # the more recent case, replace the old one with the new one if not redisClockIncluded and clockIncluded: client.hdel(key, json.dumps(redisClockDict)) client.hset(key, json.dumps(clockDict), rating) # Client never rated yet since we didn't find it in the hash so add it if isClientExist == False: client.hset(key, json.dumps(clockDict), rating) # calls method that returns the result as {rating, choices, clock} final_rating_result = get_final_rating_result(key) return final_rating_result
def get(id, ec=False, port=lb_base): """ Get a value. By default, this will issue a strongly consistent read to the load balancer. Setting ec=True will request an eventually consistent read. Setting port to the port of a DB instance does a direct get to that instance, bypassing the load balancer. """ headers = { 'Accept': 'application/json' } url = endpoint(id, port) try: if ec: response = requests.get(url, headers=headers, params={'consistency': 'weak'}) else: response = requests.get(url, headers=headers) except Exception as e: raise Exception("Invalid request: url %s, exception %s" % (url, e)) try: data = response.json() except: raise Exception('Unexpected response: %s HTTP %d %s' % (url, response.status_code, response.text)) try: rating = float(data['rating']) except: rating = data['rating'] choices = data['choices'] #TODO: Handle return of malformed vector clock clocks = data['clocks'] return rating, choices, [VectorClock.fromDict(vcstr) for vcstr in clocks]
def put_rating(entity): # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) # Parse the request data = json.load(request.body) setrating = data.get('rating') setclock = VectorClock.fromDict(data.get('clock')) key = '/rating/'+entity #key = '/rating-and-clock/'+entity #sync_with_neighbour_queue(key) merge_with_db(setrating, setclock, key) sync_with_neighbour_queue(key) # lets grab the results of our work! result = get_final_rating_result(key) #final_rating_key = '/rating/'+entity #client.hset(final_rating_key, 'rating', result["rating"]) # Return rating return { "rating": result["rating"] }
def get_rating(entity): print "Called GET on ENTITY", entity key = '/rating/'+entity hashed_entity = hash(entity) db_index = hashed_entity % ndb print "entity primary is", db_index #Calculate rating clocks = {} choices = 0 #print "TABLE", table print "doing GOSSIP for read:", clocks gossip_protocol() table = client.hgetall(key) clocks = table.keys() choices = table.values() print "CLOCKS", clocks print "CHOICES:", choices print "TYPE OF CLOCKS", type(clocks) total = 0 for i in range (len(choices)): choices[i] = float(choices[i]) total = total + choices[i] clocksArr = [] for clock in clocks: print "CLOCK", type(clock) print "CLOCK", clock clock = VectorClock.fromDict(ast.literal_eval(clock)) clocksArr.append(clock.asDict()) print "TYPE OF CLOCK", type(clock.asDict()) print "CLOCKSARR", clocksArr checkDigestList() if (total > 0): finalrating = float(total/len(choices)) return { "rating": finalrating, "choices": choices, "clocks": clocksArr } elif (len(clocksArr) > 0): return { "rating": 0, "choices": choices, "clocks": clocksArr} else: return { "rating": 0, "choices": [], "clocks": []}
def get(self, channel): resp = requests.get('http://localhost:'+str(self.port)+'/q/'+channel, headers={'content-type': 'application/json'}) jresp = resp.json() if len(jresp) > 0: for k in jresp: if (isinstance(jresp[k],dict) and jresp[k].keys() == [CLOCK_CODE]): jresp[k] = [VectorClock.fromDict(dc) for dc in jresp[k][CLOCK_CODE]] return jresp else: return None
def process_request_data(entity, rating, newclocks): print "processing request data" # Weave the new rating into the current rating list ratingkey = "" if "rating" not in entity: ratingkey = '/rating/'+entity else: ratingkey = entity spectator = True newClockList = [] table = client.hgetall(ratingkey) clocks = table.keys() choices = table.values() print "table", table print "clocks", clocks print "choices", choices print "newclocks", newclocks print "type of newclocks", type(newclocks) if (len(clocks) == 0): newClockList.append(newclocks) print "appending new clock to our clock list because our clock list is empty. NewClockList:", newClockList print "type of newclocks", type(newclocks) for entry in clocks: entry = VectorClock.fromDict(ast.literal_eval(entry)) if (entry < newclocks): print "Our current clock is older!", entry, newclocks spectator = False print "STORING IN REDIS DATABASE", newclocks.asDict(), rating client.hset(ratingkey, newclocks.asDict(), rating) client.hdel(ratingkey, entry.asDict()) newClockList.append(newclocks) elif (newclocks <= entry): spectator = False if(spectator): print "spectator", spectator for clock in newClockList: print type(clock) print "STORING IN REDIS DATABASE", ratingkey, clock.asDict(), rating client.hset(ratingkey, clock.asDict(), rating) # Return the new rating for the entity return rating, newClockList
def put_rating(entity): # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) # Parse the request data = json.load(request.body) rating = data.get('rating') clock = VectorClock.fromDict(data.get('clock')) # Basic sanity checks on the rating if isinstance(rating, int): rating = float(rating) if not isinstance(rating, float): return abort(400) # YOUR CODE HERE # HASH THE ENTITY TO DETERMINE ITS SHARD hashed_entity = hash(entity) # find out which database process this entity hashes to db_index = hashed_entity % ndb print "ENTITY has PRIMARY DB as", db_index, entity # change the dbBasePort to be the port on which the hashed database is talking on dbBasePort = 3000 dbBasePort = dbBasePort + db_index # PUT THE PORT FOR THE CORRECT SHARD IN url below url = 'http://localhost:'+str(dbBasePort)+'/rating/'+entity print "URL", url # RESUME BOILERPLATE CODE... # Update the rating res = requests.put(url, data=json.dumps({'rating': rating, 'clock': clock.asDict()}), headers={'content-type': 'application/json'} ) # Return the new rating for the entity return { "rating": res.json()['rating'] }
def get_from_redis(tea_name): try: data = eval(client.get("tea:%s:json" % tea_name)) except: #print "*** Error [get from redis] no key found for the tea:" + tea_name return None,None,None try: rating = data["rating"] except: rating = data["rating"] choices = data["choices"] clocks = data["clocks"] #vc.clock, ex) {'c1': 10, 'c0': 7} return rating, choices, VectorClock.fromDict(clocks)
def put_rating(entity): print "Called PUT on ENTITY", entity hashed_entity = hash(entity) db_index = hashed_entity % ndb print "entity primary is", db_index # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) # Parse the request data = json.load(request.body) setrating = data.get('rating') setclock = VectorClock.fromDict(data.get('clock')) key = '/rating/'+ entity finalrating = 0 newClockList = [] print "ENTITY", entity print "setrating", setrating print "setclock", setclock print "write rating to REDIS Database:", entity, setrating, setclock (finalrating, newClockList) = process_request_data(entity, setrating, setclock) print "newClockList:", newClockList # record change in digest list global digest_list print "record change in digest_list:", id, key, finalrating, setclock for clock in newClockList: digest_list.append({"id": id, "key": key, "rating": finalrating, "clock": clock}) gossip_protocol() checkDigestList() # Return rating return { "rating": finalrating }
def put_rating(entity): # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) data = json.load(request.body) if DEBUG: print "[put_rating]", data recieved_rating = data.get('rating') recieved_vc = VectorClock.fromDict(data.get('clocks')) # Basic sanity checks on the rating if isinstance(recieved_rating, int):recieved_rating = float(recieved_rating) if not isinstance(recieved_rating, float): return abort(400) # Weave the new rating into the current rating list key = '/rating/'+entity tea_name = entity # COMPUTE THE MEAN, finalrating after converge existing and recieving value finalrating, choices, new_vc_list = vector_converge(tea_name,recieved_rating,recieved_vc) # SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY if choices!=None: put_to_redis(tea_name, finalrating,choices,new_vc_list) #store new score # YOUR CODE HERE # MERGE WITH CURRENT VALUES FOR THIS KEY # REPLACE FOLLOWING WITH CORRECT FINAL RATING # finalrating = 0.0 # SAVE NEW VALUES # GOSSIP # Return rating return { "rating": finalrating }
def get(id): headers = { 'Accept': 'application/json' } url = endpoint+'/rating/'+id try: request = requests.get(url, headers=headers) data = request.json() except: raise Exception('Invalid request: %s HTTP %d %s' % (url, request.status_code, request.text)) try: rating = float(data['rating']) except: rating = data['rating'] choices = json.loads(data['choices']) #TODO: Handle return of malformed vector clock clocks = json.load(StringIO.StringIO(data['clocks'])) return rating, choices, [VectorClock.fromDict(vcstr) for vcstr in clocks]
def put_rating(entity): # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) # Parse the request data = json.load(request.body) rating = data.get('rating') clock = VectorClock.fromDict(data.get('clock')) query_param_dict = parse_qs(urlparse(request.url).query, keep_blank_values=True) # Basic sanity checks on the rating if isinstance(rating, int): rating = float(rating) if not isinstance(rating, float): return abort(400) # WE ARE NOT SUPPOSE TO HAVE EVENTUAL CONSISTENT WRITES. What does that even mean... #if 'consistency' in query_param_dict.keys() and query_param_dict['consistency'] == 'weak': # shard_number = get_shard_number(entity) #else: # shard_number = get_shard_number(entity, consistent=True) dbBasePort = get_shard_number(entity, consistent=True) url = 'http://localhost:'+str(dbBasePort)+'/rating/'+entity res = requests.put(url, data=json.dumps({'rating': rating, 'clock': clock.asDict()}), headers={'content-type': 'application/json'}) # Return the new rating for the entity return { "rating": res.json()['rating'] }
def gossip_protocol(): # first check the channel to see if there is anything in there isItemInChannel = True newClockList = [] global queue global id global digest_list while (isItemInChannel): #TODO: remember do this only if ndb > 1 msg = [] if (id == 0): msg = queue.get(str(ndb - 1)) if (not msg): isItemInChannel = False else: msg = queue.get(str(id - 1)) if (not msg): isItemInChannel = False print "isItemInChannel", isItemInChannel if (isItemInChannel): print "MSG", msg if (id != msg['id']): ratingValue = msg['rating'] key = msg['key'] clock = msg['clock'] # turn the clock retrieved off channel into a vector clock type clock = VectorClock.fromDict(clock) # TODO should I be grabbing the clock from the msg and storing that in REDIS print "writing this key retrieved off the channel to REDIS", key, ratingValue, clock (ratingValue, newClockList) = process_request_data(key, ratingValue, clock) print "append these values to this instance's digest_list, for later gossip to its neighbour:", ratingValue, clock digest_list.append({"id": id, "key": key, "rating": ratingValue, "clock": clock})
def put_rating(entity): # Check to make sure JSON is ok type = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not type: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', type) # Read the data sent from the client data = json.load(request.body) recieved_rating = data.get('rating') recieved_vc = VectorClock.fromDict(data.get('clocks')) # Basic sanity checks on the rating if isinstance(recieved_rating, int):recieved_rating = float(recieved_rating) if not isinstance(recieved_rating, float): return abort(400) # Weave the new rating into the current rating list key = '/rating/'+entity tea_name = entity # COMPUTE THE MEAN, finalrating after converge existing and recieving value finalrating, choices, new_vc = vector_converge(tea_name,recieved_rating,recieved_vc) # SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY if choices!=None: put_to_redis(tea_name, finalrating,choices,new_vc) #store new score # Return the new rating for the entity return { "rating": finalrating }
def put_rating(entity): # Check to make sure JSON is ok mimetype = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not mimetype: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', mimetype) # Parse the request data = json.load(request.body) rating = data.get('rating') clock = VectorClock.fromDict(data.get('clock')) # Basic sanity checks on the rating if isinstance(rating, int): rating = float(rating) if not isinstance(rating, float): return abort(400) # YOUR CODE HERE # HASH THE ENTITY TO DETERMINE ITS SHARD # PUT THE PORT FOR THE CORRECT SHARD IN url below url = 'http://localhost:'+str(dbBasePort+hashEntity(entity, ndb))+'/rating/'+entity # RESUME BOILERPLATE CODE... # Update the rating res = requests.put(url, data=json.dumps({'rating': rating, 'clock': clock.asDict()}), headers={'content-type': 'application/json'}) # Return the new rating for the entity return { "rating": res.json()['rating'] }
def put_rating(entity): # Check to make sure JSON is ok type = mimeparse.best_match(['application/json'], request.headers.get('Accept')) if not type: return abort(406) # Check to make sure the data we're getting is JSON if request.headers.get('Content-Type') != 'application/json': return abort(415) response.headers.append('Content-Type', type) # Read the data sent from the client data = json.load(request.body) setrating = data.get('rating') setclock = VectorClock.fromDict(data.get('clocks')) # Basic sanity checks on the rating if isinstance(setrating, int): setrating = float(setrating) if not isinstance(setrating, float): return abort(400) # Weave the new rating into the current rating list key = '/rating/'+entity # YOUR CODE GOES HERE # REPLACE THE FOLLOWING LINE # HINT: CONSIDER USING THE HASH DATA TYPE IN REDIS (HGET/HSET/...) old_rating = client.hget(key, 'rating') # if rating does not exist, add it. Otherwise.. # SET THE RATING, CHOICES, AND CLOCKS IN THE DATABASE FOR THIS KEY # COMPUTE THE MEAN, finalrating if not old_rating: client.hset(key, 'rating', setrating) client.hset(key, 'choices', [setrating]) client.hset(key, 'clocks', jsonify_vcl([setclock])) finalrating = setrating else: finalrating = old_rating choices = eval(client.hget(key, 'choices')) vcl = eval(client.hget(key, 'clocks')) new_vcl = [] new_choices = [] greaterThanAlreadyFound = False needToUpdateDB = True for i in range(0, len(vcl)): old_clock = VectorClock.fromDict(vcl[i]) # if the received clock is older, nothing needs updating if setclock <= old_clock: needToUpdateDB = False break else: # if the received clock is newer, make changes accordingly if setclock > old_clock: # If we have not found an older clock and replaced it with the # new one previously, put this new clock in. Otherwise, ignore. if not greaterThanAlreadyFound: greaterThanAlreadyFound = True new_vcl.append(setclock) new_choices.append(setrating) # incomparable else: new_vcl.append(old_clock) new_choices.append(choices[i]) # Update DB only if the received clock is not older than or the same as any of the # existing clocks if needToUpdateDB: # if the received clock is not newer than any of the existing clocks, it's # incomparable if not greaterThanAlreadyFound: new_vcl.append(setclock) new_choices.append(setrating) # calculate the new rating ratingSum = 0.0 for choice in new_choices: ratingSum+=choice finalrating = ratingSum/len(new_choices) # update DB client.hset(key, 'rating', finalrating) client.hset(key, 'choices', new_choices) client.hset(key, 'clocks', jsonify_vcl(new_vcl)) # Return the new rating for the entity return { "rating": finalrating }
def merge_dict(vc_list): json_dict = "{%s}" % ", ".join(["%s" % vc.asItemString() for vc in vc_list ]) return VectorClock.fromDict(eval(json_dict))
def convert2vc_list(json_clocks_list): return [VectorClock.fromDict(clocks) for clocks in eval(json_clocks_list)]