def do_DELETE(self):
master_const.check_start = True
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
parser_post_type_obj = UrlParser('delete')
dict_return = parser_post_type_obj.parse(self.path)
# Destroy table
if dict_return["function_name"] == master_const.del_function_types[0]:
if dict_return["table_name"] not in master_const.table_names['tables']:
self._set_response(404)
return
if dict_return["table_name"] in master_const.locks and len(
master_const.locks[dict_return["table_name"]]) != 0:
self._set_response(409)
return
for tablet in master_const.server_table_dict:
if dict_return["table_name"] in master_const.server_table_dict[tablet]:
tablet_name = tablet
url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"],
"/Delete/" + str(dict_return["table_name"] + '/'))
print(url_tablet)
response = requests.delete(url_tablet)
if (response.status_code == 200):
master_const.table_names["tables"].remove(dict_return["table_name"])
master_const.server_load_dict[tablet_name] -= 1
master_const.server_table_dict[tablet_name].remove(dict_return["table_name"])
del master_const.table_info[dict_return["table_name"]]
self._set_response(response.status_code)
return
# CLose the lock
elif dict_return["function_name"] == master_const.del_function_types[1]:
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names["tables"]:
self._set_response(404)
return
if dict_return["table_name"] in master_const.locks and client_data["client_id"] not in master_const.locks[
dict_return["table_name"]]:
self._set_response(400)
return
master_const.locks[dict_return["table_name"]].remove(client_data["client_id"])
self._set_response(200)
def check_tablets():
count = 0
while True:
if master_const.check_start and len(master_const.running_tablets) > 0:
tablet_name = master_const.running_tablets[count % len(master_const.running_tablets)]
url_check = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"],
"/Check/")
try:
response = requests.post(url_check, timeout=5)
except:
print(tablet_name + " is down!")
master_const.running_tablets.remove(tablet_name)
# Another tablet server takes control
another_tablet_name = master_const.running_tablets[count % len(master_const.running_tablets)]
# print(another_tablet_name + " stands out to save the world!")
print(another_tablet_name + " takes control!")
url_recover = MasterSupport.url(master_const.tablets_info[another_tablet_name]["hostname"],
master_const.tablets_info[another_tablet_name]["port"],
"/Recover/")
requests.post(url_recover, json={"hostname": master_const.tablets_info[another_tablet_name]["hostname"],
"port": master_const.tablets_info[tablet_name]["port"]})
master_const.server_load_dict[another_tablet_name] += master_const.server_load_dict[tablet_name]
master_const.server_load_dict[tablet_name] = 0
for table_name in master_const.server_table_dict[tablet_name]:
master_const.table_info[table_name]["tablets"].remove(
{"hostname": master_const.tablets_info[tablet_name]["hostname"],
"port": master_const.tablets_info[tablet_name]["port"]})
if {"hostname": master_const.tablets_info[another_tablet_name]["hostname"],
"port": master_const.tablets_info[another_tablet_name]["port"]} not in \
master_const.table_info[table_name]["tablets"]:
master_const.table_info[table_name]["tablets"].append(
{"hostname": master_const.tablets_info[another_tablet_name]["hostname"],
"port": master_const.tablets_info[another_tablet_name]["port"]})
master_const.server_table_dict[another_tablet_name] += master_const.server_table_dict[tablet_name]
master_const.server_table_dict[tablet_name] = []
print("recovery done")
# print("Hero needs good rest...")
time.sleep(100)
time.sleep(5)
count += 1
def forward_to_master(self, data, table_name, request_type):
url_master = MasterSupport.url(const.master_info["master_hostname"],
const.master_info["master_port"],
"/forward/")
forward_dict = {}
forward_dict["data"] = data
forward_dict["table_name"] = table_name
if (request_type == "insert"):
response = requests.post(url_master, json=self.forward_dict)
elif request_type == "retrieve":
response = requests.get(url_master, json=self.forward_dict)
def do_POST(self):
# example: reading content from HTTP request
if not master_const.check_start:
master_const.check_start = True
try:
x = threading.Thread(target=check_tablets)
x.start()
except KeyboardInterrupt:
pass
except:
print("Error: unable to start thread")
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# Create a table
# Picks the least loaded tablet server
if dict_return["function_name"] == master_const.post_function_types[0]:
if table_dict["name"] in master_const.table_names["tables"]:
self._set_response(409)
return
tablet_name = master_const.master_operation.load_balance()
url_tablet = MasterSupport.url(
master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"], "/Create/")
hostandport = {}
hostandport["hostname"] = master_const.tablets_info[tablet_name][
"hostname"]
hostandport["port"] = master_const.tablets_info[tablet_name][
"port"]
response = requests.post(url_tablet, json=table_dict)
if (response.status_code == 200):
master_const.table_names["tables"].append(table_dict["name"])
master_const.server_load_dict[tablet_name] += 1
master_const.server_table_dict[tablet_name].append(
table_dict["name"])
master_const.table_info[table_dict["name"]] = {}
master_const.table_info[
table_dict["name"]]["name"] = table_dict["name"]
master_const.table_info[table_dict["name"]]["tablets"] = []
master_const.table_info[table_dict["name"]]["tablets"].append(
hostandport)
self._set_response(response.status_code)
# insert cells
# elif dict_return["function_name"] == master_const.post_function_types[1]:
#
# if table_dict["name"] in master_const.table_names["tables"]:
# self._set_response(409)
# return
#
# tablet_name = master_const.master_operation.load_balance()
# url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
# master_const.tablets_info[tablet_name]["port"], "/Create/")
# hostandport = {}
#
# hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
# hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
# response = requests.post(url_tablet, json=table_dict)
#
# self._set_response(response.status_code)
# Open a lock to table
if dict_return["function_name"] == master_const.post_function_types[3]:
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names[
"tables"]:
self._set_response(404)
return
if dict_return["table_name"] not in master_const.locks:
master_const.locks[dict_return["table_name"]] = []
if client_data["client_id"] in master_const.locks[
dict_return["table_name"]]:
self._set_response(400)
else:
master_const.locks[dict_return["table_name"]].append(
client_data["client_id"])
self._set_response(200)
# tablet start
elif dict_return["function_name"] == master_const.post_function_types[
4]:
client_data = json.loads(post_data)
tablet_hostname = client_data["hostname"]
tablet_port = client_data["port"]
tablet_name = "tablet" + str(len(master_const.running_tablets) + 1)
master_const.running_tablets.append(tablet_name)
master_const.tablets_info.update({
tablet_name: {
"hostname": tablet_hostname,
"port": tablet_port
}
})
self._set_response(200)
# sharding
elif dict_return["function_name"] == master_const.post_function_types[
5]:
client_data = json.loads(post_data)
table_name = client_data["table_name"]
row_from_1 = client_data["row_from_1"]
row_to_1 = client_data["row_to_1"]
row_from_2 = client_data["row_from_2"]
row_to_2 = client_data["row_to_2"]
tablet_hostname = client_data["hostname"]
tablet_port = client_data["port"]
new_tablet_name = ""
new_tablet_hostname = ""
new_tablet_port = ""
for running_tablet_name in master_const.running_tablets:
if not master_const.tablets_info[running_tablet_name][
"hostname"] == tablet_hostname or not master_const.tablets_info[
running_tablet_name]["port"] == tablet_port:
new_tablet_name = running_tablet_name
new_tablet_hostname = master_const.tablets_info[
running_tablet_name]["hostname"]
new_tablet_port = master_const.tablets_info[
running_tablet_name]["port"]
break
for running_tablet_name in master_const.running_tablets:
if master_const.tablets_info[running_tablet_name][
"hostname"] == tablet_hostname and master_const.tablets_info[
running_tablet_name]["port"] == tablet_port:
tablet_name = running_tablet_name
if new_tablet_name == "":
# no other tablets
self._set_response(400)
return
# assume we only shard once
if len(master_const.table_info[table_name]["tablets"]) > 1:
self._set_response(401)
return
master_const.table_info[table_name]["tablets"][0].update({
"row_from":
row_from_1,
"row_to":
row_to_1
})
master_const.table_info[table_name]["tablets"].append({
"hostname":
new_tablet_hostname,
"port":
new_tablet_port,
"row_from":
row_from_2,
"row_to":
row_to_2
})
url = MasterSupport.url(new_tablet_hostname, new_tablet_port,
"/Sharding/")
requests.post(url, json=client_data["data"])
self._set_response(200)
# forward
elif dict_return["function_name"] == master_const.post_function_types[
6]:
client_data = json.loads(post_data)
table_name = client_data["table_name"]
table_info = master_const.table_info[table_name]["tablets"][1]
if client_data["data"]["row"] > table_info["row_to"]:
table_info["row_to"] = client_data["data"]["row"]
url = MasterSupport.url(new_tablet_hostname, new_tablet_port,
"/api/table/" + table_name + "/cell")
requests.post(url, json=client_data["data"])
self._set_response(200)
def insert(self, table_name, post_data, recover):
if table_name not in self.manifest["table_names"]["tables"]:
return 404
cell = {}
try:
cell = json.loads(post_data)
except:
print(post_data)
return 400
if cell == {}:
return 400
if not self.find_column_family_and_column(
table_name, cell["column_family"], cell["column"], recover):
return 400
# Write to WAL
dic = {}
dic["cell"] = cell
dic["table_name"] = table_name
self.WAL.append(dic)
with open(self.WAL_filename, 'wb') as outfile:
pickle.dump(self.WAL, outfile)
if table_name not in self.mem_table:
self.mem_table[table_name] = {}
if cell["column_family"] not in self.mem_table[table_name]:
self.mem_table[table_name][cell["column_family"]] = {}
if cell["column"] not in self.mem_table[table_name][
cell["column_family"]]:
self.mem_table[table_name][cell["column_family"]][
cell["column"]] = OOBTree()
try:
if self.sharded == True:
if self.shard_info["data"]["table_name"] == table_name:
if self.shard_info["data"]["column_family"] == cell[
"column_family"]:
if self.shard_info["data"]["column"] == cell["column"]:
self.forward_to_master(self, cell, table_name,
"insert")
return 200
except:
pass
t = self.mem_table[table_name][cell["column_family"]][cell["column"]]
if cell["row"] not in t:
t.update({cell["row"]: {"row": cell["row"], "data": cell["data"]}})
else:
t[cell["row"]]["data"] += cell["data"]
# Garbage collection
while len(t[cell["row"]]["data"]) > 5:
t[cell["row"]]["data"].pop(0)
# Send sharding request to master
if (len(t) > self.external_shard):
self.shard_info["data"]["ssindex"] = self.manifest["ssindex2"]
self.sharded = True
url_master = MasterSupport.url(
const.master_info["master_hostname"],
const.master_info["master_port"], "/sharding/")
response = requests.post(url_master, json=self.shard_info)
print("response over")
self.shard_info["row_from_1"] = 0
self.entries = self.entries + 1
if self.entries == self.max_entries:
self.spill_to_disk()
return 200
def do_POST(self):
# example: reading content from HTTP request
if not master_const.check_start:
master_const.check_start = True
try:
x = threading.Thread(target=check_tablets)
x.start()
except KeyboardInterrupt:
pass
except:
print("Error: unable to start thread")
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# Create a table
# Picks the least loaded tablet server
if dict_return["function_name"] == master_const.post_function_types[0]:
if table_dict["name"] in master_const.table_names["tables"]:
self._set_response(409)
return
tablet_name = master_const.master_operation.load_balance()
url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"], "/Create/")
hostandport = {}
hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
response = requests.post(url_tablet, json=table_dict)
if (response.status_code == 200):
master_const.table_names["tables"].append(table_dict["name"])
master_const.server_load_dict[tablet_name] += 1
master_const.server_table_dict[tablet_name].append(table_dict["name"])
master_const.table_info[table_dict["name"]] = {}
master_const.table_info[table_dict["name"]]["name"] = table_dict["name"]
master_const.table_info[table_dict["name"]]["tablets"] = []
master_const.table_info[table_dict["name"]]["tablets"].append(hostandport)
self._set_response(response.status_code)
# insert cells
# elif dict_return["function_name"] == master_const.post_function_types[1]:
#
# if table_dict["name"] in master_const.table_names["tables"]:
# self._set_response(409)
# return
#
# tablet_name = master_const.master_operation.load_balance()
# url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
# master_const.tablets_info[tablet_name]["port"], "/Create/")
# hostandport = {}
#
# hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
# hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
# response = requests.post(url_tablet, json=table_dict)
#
# self._set_response(response.status_code)
# Open a lock to table
if dict_return["function_name"] == master_const.post_function_types[3]:
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names["tables"]:
self._set_response(404)
return
if dict_return["table_name"] not in master_const.locks:
master_const.locks[dict_return["table_name"]] = []
if client_data["client_id"] in master_const.locks[dict_return["table_name"]]:
self._set_response(400)
else:
master_const.locks[dict_return["table_name"]].append(client_data["client_id"])
self._set_response(200)