def gossip():
# checks if I am currently gossiping with someone else
if not shard.gossiping:
shard.lastToGossip = False
incoming_data = json.loads(request.get_json())
shard.gossiping = True
# causal context of the incoming node trying to gossip
other_context = incoming_data["causal-context"]
# key store of incoming node trying to gossip
other_kvstore = incoming_data["kv-store"]
# this is true if the other node determined i will be the tiebreaker
tiebreaker = True if incoming_data["tiebreaker"] == ADDRESS else False
incoming_Vc = VectorClock(view=None, clock=other_context)
if other_kvstore == shard.keystore:
shard.gossiping = False
return jsonify({"message": "We're equal."}), 201
elif incoming_Vc.allFieldsZero():
shard.gossiping = False
return jsonify({"message": "You dont have any data"}), 201
elif incoming_Vc.selfHappensBefore(shard.VC.__repr__()):
# I am at least concurrent or after
shard.gossiping = False
return jsonify({"message": "I Don't need yours."}), 201
elif incoming_Vc.__repr__() != shard.VC.__repr__():
shard.keystore = other_kvstore
shard.VC.merge(other_context, ADDRESS)
shard.gossiping = False
return jsonify({"message": "I took your data"}), 200
return jsonify({"message": "gossiping"}), 201
def __init__(self, router, address, view, replication_factor):
KV_store.__init__(self, address)
self.gossipScheduled = False
self.lastToGossip = False
self.gossiping = False
#self.sched = Scheduler()
#self.sched.start()
self.history = [('Initialized', datetime.now())]
self.ADDRESS = address
self.VC = VectorClock(view=view, clock=None)
self.ring_edge = 691 if len(
view) < 100 else 4127 # parameter for hash mod value
self.repl_factor = replication_factor
self.num_shards = 0
self.virtual_range = 16
self.shard_interval = self.ring_edge // self.virtual_range
self.nodes = []
self.shard_ID = -1
self.V_SHARDS = [] # store all virtual shards
self.P_SHARDS = [[] for i in range(0, self.num_shards)
] # map physical shards to nodes
self.virtual_translation = {} # map virtual shards to physical shards
self.backoff_mod = 59
self.router = router
self.view_change(view, replication_factor)
def state_transfer():
data = request.get_json()
other_vector_clock = data["context"]
if shard.VC.selfHappensBefore(other_vector_clock):
shard.keystore = data["kv-store"]
shard.vc = VectorClock(view=None, clock=other_vector_clock)
return {"message": "Acknowledged"}, 201
def put(self, key, metadata, value, destnode=None):
if destnode is None:
destnode = random.choice(DynamoNode.nodelist)
if len(metadata) == 1 and metadata[0] is None:
metadata = VectorClock()
else:
metadata = VectorClock.converge(metadata)
putmsg = ClientPut(self, destnode, key, value, metadata)
Framework.send_message(putmsg)
return putmsg
def __init__(self):
self.servers = []
self.rm = None
# Initial selection of replica manager to communicate with
try:
self.rm = self._choose_replica()
except ValueError as e:
print(e)
self.ts = VectorClock(REPLICA_NUM) # Vector timestamp of front end
def put(self, key, metadata, value, destnode=None):
if destnode is None: # Pick a random node to send the request to
destnode = random.choice(DynamoNode.nodelist)
# Input metadata is always a sequence, but we always need to insert a
# single VectorClock object into the ClientPut message
if len(metadata) == 1 and metadata[0] is None:
metadata = VectorClock()
else:
# A Put operation always implies convergence
metadata = VectorClock.converge(metadata)
putmsg = ClientPut(self, destnode, key, value, metadata)
Framework.send_message(putmsg)
return putmsg
def __init__(self, replica_id, stopper, status=None):
super().__init__()
self._id = replica_id
# Replica status properties
self.failure_prob = 0.1
self.overload_prob = 0.2
self.auto_status = True
if status not in [n.value for n in list(Status)]:
print('Invalid status provided, defaulting to active.')
self.status = Status.ACTIVE
else:
self.status = Status(status)
self.auto_status = False
print(f'Status set to {status}.',
'Automatic status updating disabled.')
# Gossip Architecture State
self.value_ts = VectorClock(REPLICA_NUM) # aka data timestamp
self.replica_ts = VectorClock(REPLICA_NUM) # aka log timestamp
self.update_log = []
self.ts_table = [
VectorClock(REPLICA_NUM) if i != self._id else None
for i in range(REPLICA_NUM)
]
self.executed = []
self.pending_queries = queue.Queue()
self.query_results = {}
self.interval = 8.0 # interval between gossip exchanges
self.other_replicas = self._find_replicas()
self.stopper = stopper # Used to indicate to server to stop
# Locks for objects shared between threads
self.vts_lock = threading.Lock() # for value_ts
self.rts_lock = threading.Lock() # for replica_ts
self.log_lock = threading.Lock() # for update_log
def gossip():
# checks if I am currently gossiping with someone else
if not shard.gossiping:
incoming_data = json.loads(request.get_json())
shard.gossiping = True
# causal context of the incoming node trying to gossip
other_context = incoming_data["context"]
# key store of incoming node trying to gossip
other_kvstore = incoming_data["kv-store"]
if other_kvstore == shard.keystore:
return jsonify({"message": "We're equal."}), 200
# this is true if the other node determined i will be the tiebreaker
tiebreaker = True if incoming_data["tiebreaker"] == ADDRESS else False
incoming_Vc = VectorClock(view=None, clock=other_context)
if shard.VC.selfHappensBefore(other_context):
# I am before
# i accept your data
shard.keystore = other_kvstore
shard.VC.merge(other_context, ADDRESS)
shard.gossiping = False
print("I HAPPENED BEFORE, I TAKE YOU" + str(shard.keystore),
file=sys.stderr)
return jsonify({"message": "I took your data"}), 200
elif incoming_Vc.selfHappensBefore(shard.VC.__repr__()):
# I am after the incoming one, so return my data
shard.gossiping = False
return jsonify({
"message": "I am after you, take my data",
"context": shard.VC.__repr__(),
"kv-store": shard.keystore,
}), 501
elif tiebreaker:
shard.gossiping = False
return jsonify({
"message": "I am the tiebreaker, take my data",
"context": shard.VC.__repr__(),
"kv-store": shard.keystore,
}), 501
elif not tiebreaker:
if bool(other_kvstore) and not incoming_Vc.allFieldsZero():
shard.keystore = other_kvstore
shard.VC.merge(other_context, ADDRESS)
shard.gossiping = False
print("I DID NOT HAPPEN BEFORE BUT AM NOT THE TIEBREAKER" +
str(shard.keystore),
file=sys.stderr)
return jsonify({"message": "I took your data"}), 200
shard.gossiping = False
return jsonify({"message": "I am gossiping with someone else"}), 400
def checkList(msg, host): temp = VectorClock() data = getClock(host) for (i,j) in data: temp.update(i,j) for (index,msgi) in enumerate(msg): aux = Vector.getCounter(str(index)) try: aux_1 = temp.getCounter(str(index)) Vector.update(str(index),int(aux_1)) except: continue for (i,mes) in enumerate(messages): if i == index: mes[0] = msgi[0]
def gossip(self):
if (gossiping == False):
gossiping = True
replica_ip_addresses = self.shard_replicas(self.shard_ID)
replica_index = random.randint(0, len(replica_ip_addresses) - 1)
while (self.shard_ID == replica_index):
replica_index = random.randint(0,
len(replica_ip_addresses) - 1)
replica = replica_ip_addresses[replica_index]
tiebreaker = replica if (
replica_index > self.shard_ID) else self.ADDRESS
data = {
"context": self.VC.__repr__(),
"kv-store": self.keystore,
"tiebreaker": tiebreaker
}
content, code = self.router.PUT(replica,
'/kv-store/internal/gossip/', data,
False)
if (code == 200):
# 200: They took my data
gossiping = False
elif (code == 501):
# 501:
# the other node was either the tiebreaker or happened after self
# so this node takes its data
# context of node
other_context = content["context"]
# key store of incoming node trying to gossip
other_kvstore = content["kv-store"]
self.VC = VectorClock(view=None, clock=other_context)
self.keystore = other_kvstore
#self happened before other, take its kvstore and merge with my clock
# concurrent but other is tiebreaker
else:
# 400: Other is already gossiping with someone else
# ELSE: unresponsive node (maybe itll be code 404?)
gossiping = False
else:
# Curretly gossiping,
# Will call after gossip backoff again
gossiping = False
def __init__(self, ID, IP, servers_list):
super(Server, self).__init__()
self.id = int(ID)
self.ip = str(IP)
self.servers_list = servers_list
self.serverIndex = self.servers_list.index(self.ip)
self.vectorClock = VectorClock(self.serverIndex, len(self.servers_list))
self.myLogger = mylogger.Logger(self.ip)
self.blackboard = distributedboard.Blackboard(self.vectorClock, self.myLogger)
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
# list all REST URIs
# if you add new URIs to the server, you need to add them here
# API for Clients
self.route('/', callback=self.index)
self.get('/board', callback=self.get_board)
self.post('/', callback=self.post_index)
self.post('/board', callback=self.post_board)
self.post('/board/<number>/', callback=self.post_board_ID)
# API for Scripts
self.get('/serverlist', callback=self.get_serverlist)
self.get('/board/alldata',callback=self.get_board_data)
# API for Server Internals
self.post('/propagate', callback=self.post_propagate)
self.post('/update_ctrl', callback=self.post_update_ctrl)
# we give access to the templates elements
self.get('/templates/<filename:path>', callback=self.get_template)
# You can have variables in the URI, here's an example
# self.post('/board/<element_id:int>/', callback=self.post_board) where post_board takes an argument (integer) called element_id
# Lab3 Option Task!!!!
self.get('/operation_log_size', callback=self.get_operation_log_size)
thread = Thread(target=self.checkUpdatesOfOtherServers)
thread.start()
def gossip(self):
if (self.gossiping == False):
current_key_store = self.keystore
self.gossiping = True
replica_ip_addresses = self.shard_replicas(self.shard_ID)
replica = replica_ip_addresses[(random.randint(
0,
len(replica_ip_addresses) - 1))]
while (self.ADDRESS == replica):
replica = replica_ip_addresses[(random.randint(
0,
len(replica_ip_addresses) - 1))]
myNumber = int((self.ADDRESS.split(".")[3]).split(":")[0])
otherNumber = int((replica.split(".")[3]).split(":")[0])
tiebreaker = replica if (otherNumber > myNumber) else self.ADDRESS
data = {
"context": self.VC.__repr__(),
"kv-store": current_key_store,
"tiebreaker": tiebreaker
}
print("sending to node: " + replica + " " + str(data),
file=sys.stderr)
try:
response = self.router.PUT(replica,
'/kv-store/internal/gossip/',
json.dumps(data))
except:
code = -1
code = response.status_code
if (code == 200):
# 200: They took my data
self.gossiping = False
elif (code == 501):
content = response.json()
# 501:
# the other node was either the tiebreaker or happened after self
# so this node takes its data
# context of node
other_context = content["context"]
# key store of incoming node trying to gossip
other_kvstore = content["kv-store"]
incoming_Vc = VectorClock(view=None, clock=other_context)
if bool(other_kvstore) and not incoming_Vc.allFieldsZero():
if current_key_store == self.keystore:
print("I TOOK DATA: " + str(self.keystore),
file=sys.stderr)
self.VC.merge(other_context, self.ADDRESS)
self.keystore = other_kvstore
else:
print("I RECIEVED AN UPDATE WHILE GOSSIPING, ABORT",
file=sys.stderr)
self.gossip = False
#self happened before other, take its kvstore and merge with my clock
# concurrent but other is tiebreaker
else:
# 400: Other is already gossiping with someone else
# ELSE: unresponsive node (maybe itll be code 404?)
self.gossiping = False
else:
# Curretly gossiping,
# Will call after gossip backoff again
self.gossiping = False
return 200
from message_struct import MessageStruct
from client_socket import SocketDynamoClient
import logging, os
from flask import request, jsonify, Response
from flask import render_template
from consistentHashing import preference_list
from creating_bucket import create_bucket
from client_socket import bucket_creation_1, create_down_system, share_v_clock
from vectorclock import VectorClock
import json, random
peers_file_path = '/home/HDUSER/clouda2/peers.txt'
gossip_file_path = '/home/HDUSER/clouda2/gossip.txt'
vec = VectorClock()
def createApp():
resp_data = {}
app = Flask(__name__)
file_handler = logging.FileHandler('server.log')
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
PROJECT_HOME = os.path.dirname(os.path.realpath(__file__))
UPLOAD_FOLDER = '{}/uploads/'.format(PROJECT_HOME)
UPLOAD_BUCKET_FOLDER = '/home/HDUSER/clouda2/uploads/'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
if not os.path.exists(peers_file_path):
messages = []
servers_list = []
know_servers = []
layoutx = []
layouty = []
x = 0
y = 0
mylink = None
NO_ERROR = 200
Vector = VectorClock()
@get('/')
@view('index')
def index():
return {'messages': messages, 'layoutx' : layoutx, 'layouty' : layouty, 'users' : servers_list}
#@route('/<on>')
#@view('index')
#def index(name):
# return {'messages': messages, 'layoutx' : layoutx, 'layouty' : layouty,'users' : servers_list}
#Recebe a mansagem e sincroniza com a lista, atualizando também o vectorClock respectivo a cada posição da
#tabela
@post('/send')
def sendMessage():
global x
