def __init__(self, number):
self.shard_size = int(number)
self.past_shard_size = int(number)
self.views = get_array_views()
self.num_nodes = len(self.views)
self.shard_directory = {}
self.build_directory()
def read_repair(all_node_info, key):
for each in all_node_info:
print(all_node_info[each])
print("")
max = None
all_clocks = []
for k in all_node_info:
print(k)
print("")
value = all_node_info[k]
vc_ts_k = (value['payload']['vc'], value['payload']['tstamp'], k)
if max == None:
max = vc_ts_k
else:
max = find_larger_vector_or_timestamp(vc_ts_k, max)
correct_vc = max[0]
correct_ts = max[1]
correct_ip = max[2]
correct_payload = all_node_info[correct_ip]
print(correct_payload)
ip_list_to_repair = get_array_views()
ip_list_to_repair.remove(correct_ip)
payload = {
"latest_timestamp": correct_ts,
"vc": correct_vc,
"pos": get_array_views().index(correct_ip),
"causal_context": correct_payload['payload']['causal_context'],
"tstamp": correct_ts
}
if 'val' in correct_payload:
val = correct_payload['val']
data = "val=" + val + "&&payload=" + json.dumps(payload)
for i in range(0, len(ip_list_to_repair)):
url = "http://" + ip_list_to_repair[i] + "/keyValue-store/" + key
requests.put(url, data=data)
else:
val = None
return val
def put(shards, num_shards, store, should_broadcast):
response = shards.update(num_shards, store)
if should_broadcast == True:
payload_to_send = {"num": num_shards, "broadcaster": False}
ips = get_array_views()
my_ip = environ.get("IP_PORT")
for ip in ips:
if ip == my_ip:
continue
url = "http://" + ip + "/shard/changeShardNumber"
requests.put(url, data=payload_to_send)
status = None
if response["is_successful"]:
status = 200
return JsonResponse(response, status=status)
else:
status = 400
return JsonResponse(response, status=status)
def delete_handling(request, details):
clock = details["clock"]
shards = details["shards"]
store = details["store"]
body_unicode = request.body.decode("utf-8")
body = urllib.parse.parse_qs(body_unicode)
ip_port_to_delete = body["ip_port"][0]
checkpoint = None
if "checkpoint" in body:
checkpoint = body["checkpoint"][0]
result_msg = ""
msg = ""
statuscode = 200
number_of_shards = environ.get("S")
ips = get_array_views()
response_content = {
"result": "Success",
"msg": "Successfully removed %s from view" % ip_port_to_delete,
}
if ip_port_to_delete in ips:
index_to_delete = get_index_of_target_in_views(ip_port_to_delete)
delete_ip(clock, ip_port_to_delete, ips, shards)
reconstruct_shard(shards, index_to_delete, ip_port_to_delete, store)
if checkpoint is None:
broadcast(ip_port_to_delete, ips)
else:
response_content = {
"result": "Error",
"msg": ip_port_to_delete + " is not in current view",
}
statuscode = 404
return JsonResponse(response_content, status=statuscode)
def broadcast(data, method, path, filter=None):
# loop = asyncio.get_event_loop()
# loop.run_until_complete(broadcast_async(data, method, path, filter))
# return True
ip_addresses = get_array_views(filter)
# ip_addresses.remove(filter)
if method == "PUT":
ip_addresses.remove(filter)
for i in range(0, len(ip_addresses)):
url = "http://" + ip_addresses[i] + path
requests.put(url, data=data)
elif (method == "DELETE"):
ip_addresses.remove(filter)
for i in range(0, len(ip_addresses)):
url = "http://" + ip_addresses[i] + path
requests.delete(url, data=data)
elif (method == "GET"):
result = {}
for i in range(0, len(ip_addresses)):
url = "http://" + ip_addresses[i] + path
response = requests.get(url, data=data)
result[ip_addresses[i]] = response.json()
return result
def put_handling(request, details, key):
store = details["store"]
latest_timestamp = details["latest_timestamp"]
curr_node_vc = details["clock"]
shards = details["shards"]
response_content = {}
# OPTION: VALUE MISSING
if len(request.body) <= 0:
response_content["msg"] = "Error"
response_content["result"] = "Error"
response_content["error"] = "Value is missing"
return JsonResponse(response_content, status=422)
# OPTION: KEY LENGTH INVALID
if 0 < len(key) > 200:
response_content["msg"] = "Error"
response_content["result"] = "Error"
response_content["error"] = "Key not valid"
return JsonResponse(response_content, status=422)
body_unicode = request.body.decode("utf-8")
body = parse_qs(body_unicode)
payload_json = None
if "rebalance" not in body:
payload_json = val_and_payload(request.body)["payload_json"]
else:
print("HIII")
val = val_and_payload(request.body)["val"]
# OPTION: VALUE SIZE TOO BIG
if getsizeof(val) > 1024:
response_content["result"] = "Error"
response_content["msg"] = "Object too large. Size limit is 1MB"
response_content["error"] = "Key not valid"
return JsonResponse(response_content, status=422)
shard_location = None
print("Before: ", payload_json)
# OPTION: NON-EMPTY PAYLOAD (NODES COMMUNICATING)
if payload_json:
req_vc = payload_json["vc"]
req_timestamp = payload_json["tstamp"]
if (
"latest_timestamp" in payload_json
and latest_timestamp.get_timestamp() == None
):
latest_timestamp.set_timestamp(payload_json["latest_timestamp"])
else:
lt = latest_timestamp.max_timestamp(req_timestamp)
latest_timestamp.set_timestamp(lt)
req_position = int(payload_json["pos"])
# OPTION: EMPTY PAYLOAD (USER REQUEST)
else:
views = get_array_views()
req_vc = curr_node_vc.get_vc()
req_position = views.index(environ.get("IP_PORT"))
req_timestamp = time.time()
if latest_timestamp.get_timestamp() == None:
latest_timestamp.set_timestamp(req_timestamp)
payload_json["latest_timestamp"] = latest_timestamp.get_timestamp()
payload_json = {
"pos": req_position,
"tstamp": req_timestamp,
"causal_context": details["causal_context"],
"vc": req_vc,
}
details["causal_context"] = None
binary_key = sha1(key.encode())
shard_location = int(binary_key.hexdigest(), 16) % shards.get_shard_size()
# OPTION: KEY NEVER EXISTED
if not store.has_key(key):
response_content["replaced"] = False
response_content["msg"] = "Added successfully"
response_content["payload"] = payload_json
status = 201
# OPTION: KEY ALREADY EXISTS AND IS BEING REPLACED
elif store.has_key(key):
response_content["replaced"] = True
response_content["msg"] = "Updated successfully"
response_content["payload"] = payload_json
status = 200
members = shards.get_members_in_ID(shard_location)
# if in right shard
if shard_location != None:
if members != None:
rand_address = random.choice(members)
if "rebalance" in body:
store.add(key, val, {})
data = "val=" + val + "&&payload=" + json.dumps(payload_json)
response = requests.put(
"http://" + rand_address + "/keyValue-store/" + key, data=data
)
return JsonResponse(response.json(), status=response.status_code)
else:
response_content = {
"result": "Error",
"msg": "No nodes in shard " + str(shard_location),
"payload": payload_json,
}
status = 400
return JsonResponse(response_content, status=status)
else:
if "rebalance" not in body:
curr_node_vc.increment_self()
payload_json["vc"] = curr_node_vc.get_vc()
store.add(key, val, payload_json["causal_context"])
print("I AM ADDING KEY")
response_content["owner"] = shards.get_my_shard()
return JsonResponse(response_content, status=status)
return JsonResponse(response_content, status=status)
def delete_handling(request, details, key):
store = details["store"]
latest_timestamp = details["latest_timestamp"]
curr_node_vc = details["clock"]
shards = details["shards"]
response_content = {}
# OPTION: NO KEY PROVIDED
if key is None:
response_content['result'] = 'Error'
response_content['msg'] = 'Not provided'
return JsonResponse(response_content, status=422)
payload_json = val_and_payload(request.body)["payload_json"]
shard_location = None
# OPTION: NON-EMPTY PAYLOAD (NODES COMMUNICATING)
if payload_json:
req_vc = payload_json['vc']
req_timestamp = payload_json['tstamp']
if 'latest_timestamp' in payload_json and latest_timestamp.get_timestamp() == None:
latest_timestamp.set_timestamp(payload_json['latest_timestamp'])
else:
lt = latest_timestamp.max_timestamp(req_timestamp)
latest_timestamp.set_timestamp(lt)
req_position = int(payload_json['pos'])
# OPTION: EMPTY PAYLOAD (USER REQUEST)
else:
IP_PORT = environ.get("IP_PORT")
views = get_array_views()
req_vc = curr_node_vc.get_vc()
req_position = views.index(IP_PORT)
req_timestamp = time.time()
if latest_timestamp.get_timestamp() == None:
latest_timestamp.set_timestamp(req_timestamp)
payload_json['latest_timestamp'] = latest_timestamp.get_timestamp()
payload_json['vc'] = req_vc
payload_json['pos'] = req_position
payload_json['tstamp'] = req_timestamp
payload_json['causal_context'] = details["causal_context"]
details["causal_context"] = None
binary_key = sha1(key.encode())
shard_location = int(binary_key.hexdigest(),
16) % shards.get_shard_size()
status = 200
# OPTION: KEY DELETED
if not store.has_key(key) or store.get()[key]['tombstone'] == True:
response_content['result'] = "Error"
response_content['msg'] = "Key does not exist"
response_content['payload'] = payload_json
status = 404
# OPTION: KEY EXISTS, SUCCESSFULLY DELETED
else:
response_content['result'] = "Success"
response_content['msg'] = "Key deleted"
response_content['payload'] = payload_json
# if in right shard
if (shard_location != None and not (environ.get("IP_PORT") in shards.get_members_in_ID(shard_location))):
members = shards.get_members_in_ID(shard_location)
if members != None:
rand_address = random.choice(members)
data = "payload="+json.dumps(payload_json)
response = requests.delete(
"http://"+rand_address+"/keyValue-store/"+key, data=data)
return JsonResponse(response.json(), status=response.status_code)
else:
response_content = {
"result": "Error",
"msg": "No nodes in shard " + shard_location,
"payload": payload_json
}
status = 400
return JsonResponse(response_content, status=status)
# if in wrong shard
else:
# reconsider to move this
curr_node_vc.increment_index(req_position)
payload_json['vc'] = curr_node_vc.get_vc()
store.delete_key(key)
print("I AM DELETING KEY KEY")
response_content["owner"] = shards.get_my_shard()
return JsonResponse(response_content, status=status)
# #reconsider to move this
# curr_node_vc.increment_index(req_position)
return JsonResponse(response_content, status=status)
def update_view(self):
self.views = get_array_views()
self.num_nodes = len(self.views)
def get_handling(request, details, key):
store = details["store"]
latest_timestamp = details["latest_timestamp"]
curr_node_vc = details["clock"]
shards = details["shards"]
response_content = {}
payload_json = val_and_payload(request.body)["payload_json"]
# OPTION: NO KEY PROVIDED
if key == None:
response_content = {
"result": "Error",
"msg": "Not provided",
"error": "Key does not exist",
}
return JsonResponse(response_content, status=422)
if store.is_exists(key):
value = store.get_item(key)["val"]
if value == "this-is-invalid":
response_content = {
"result": "Error",
"msg": "Unable to access key: " + key,
"payload": payload_json,
}
return JsonResponse(response_content, status=400)
# OPTION: SEARCH-GET
if "/keyValue-store/search" in request.path:
response_content["result"] = "Success"
response_content["payload"] = payload_json
if store.is_exists(key):
response_content["isExists"] = True
else:
response_content["isExists"] = False
return JsonResponse(response_content, status=200)
# OPTION: EMPTY PAYLOAD (USER REQUEST), HASH KEY AND CHECK SHARD DIRECTORY
if not payload_json:
payload_json = {
"vc": curr_node_vc.get_vc(),
"pos": get_array_views().index(environ.get("IP_PORT")),
"tstamp": latest_timestamp.get_timestamp(),
"causal_context": details["causal_context"],
}
details["causal_context"] = None
data = "payload=" + json.dumps(payload_json)
if store.is_exists(key):
response_content = {
"result": "Success",
"val": store.get_item(key)["val"],
"payload": payload_json,
"owner": shards.get_my_shard()
}
status = 200
else:
response_content = {
"result": "Error",
"msg": "Key does not exist",
"payload": payload_json,
}
status = 404
shard_location = None
binary_key = sha1(key.encode())
shard_location = int(binary_key.hexdigest(),
16) % shards.get_shard_size()
# OPTION: WE'RE IN THE WRONG SHARD, REDIRECT REQUEST TO NODE WITH CORRECT SHARD
current_node_not_in_shard = shards.get_members_in_ID(
shard_location) is not None and not (environ.get(
"IP_PORT") in shards.get_members_in_ID(shard_location))
if current_node_not_in_shard:
members = shards.get_members_in_ID(shard_location)
if members != None and len(members) > 0:
rand_address = random.choice(members)
data = "payload=" + json.dumps(payload_json)
response = requests.get("http://" + rand_address +
"/keyValue-store/" + key,
data=data)
response_content = response.json()
if response.status_code == 404:
response_content = {
"result": "Error",
"msg": "Key does not exist",
"payload": payload_json,
}
else:
del response_content["owner"]
return JsonResponse(response_content,
status=response.status_code)
else:
response_content = {
"result": "Error",
"msg": "No nodes in shard " + shard_location,
"payload": payload_json,
}
status = 400
return JsonResponse(response_content, status=status)
else:
return JsonResponse(response_content, status=status)
# OPTION: NON-EMPTY PAYLOAD (NODES COMMUNICATING). WE'RE IN THE
# RIGHT CONTAINER, JUST DO NORMAL GET
else:
views = get_array_views()
# CONDITION: If store already has the key
if store.is_exists(key):
details["causal_context"] = {
"key": key,
"vc": curr_node_vc.get_vc()
}
response_content["result"] = "Success"
response_content["val"] = store.get_item(key)["val"]
# payload_json['value'] = store.get_item(key)['val']
# CONDITION: If causal context exists and requestor's vc > this vc
# if cc_key != None and curr_node_vc.greater_than_or_equal(payload_json["vc"]) is False:
# store.add(cc_key, "too_old", None)
# UP_TO_DATE_PAYLOAD = payload_json
# CONDITION: if reading an old value, return Payload out of date,
# remove val from response_content, and set status to 400
if store.get_item(key)["val"] == "too_old":
del response_content["val"]
response_content = {
"msg": "Payload out of date",
"status": 400,
"payload": UP_TO_DATE_PAYLOAD,
}
status = 400
status = 200
payload_json = {
"vc": curr_node_vc.get_vc(),
"pos": views.index(environ.get("IP_PORT")),
"tstamp": latest_timestamp.get_timestamp(),
"causal_context": details["causal_context"],
}
else:
payload_json = {
"value": None,
"vc": curr_node_vc.get_vc(),
"pos": views.index(environ.get("IP_PORT")),
"tstamp": latest_timestamp.get_timestamp(),
"causal_context": details["causal_context"],
}
response_content["payload"] = payload_json
status = 404
response_content["owner"] = shards.get_my_shard()
response_content["payload"] = payload_json
return JsonResponse(response_content, status=status)