def do_GET(self):
content_length = self.headers['content-length']
get_data = {}
if content_length != None:
content_length = int(content_length)
get_data = self.rfile.read(content_length)
# data = None
parser_get_type_obj = UrlParser('get')
dict_return = parser_get_type_obj.parse(self.path)
# perform on the dict_return
# List Tables
if dict_return["function_name"] == master_const.get_function_types[0]:
data = master_const.table_names
data_json = json.dumps(data)
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
# Get table info
elif dict_return["function_name"] == master_const.get_function_types[
1]:
if dict_return["table_name"] in master_const.table_info:
table_info = master_const.table_info[dict_return["table_name"]]
info_json = json.dumps(table_info)
self._set_response(200)
self.wfile.write(info_json.encode("utf8"))
else:
self._set_response(404)
def do_DELETE(self):
master_const.check_start = True
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
parser_post_type_obj = UrlParser('delete')
dict_return = parser_post_type_obj.parse(self.path)
# Destroy table
if dict_return["function_name"] == master_const.del_function_types[0]:
if dict_return["table_name"] not in master_const.table_names['tables']:
self._set_response(404)
return
if dict_return["table_name"] in master_const.locks and len(
master_const.locks[dict_return["table_name"]]) != 0:
self._set_response(409)
return
for tablet in master_const.server_table_dict:
if dict_return["table_name"] in master_const.server_table_dict[tablet]:
tablet_name = tablet
url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"],
"/Delete/" + str(dict_return["table_name"] + '/'))
print(url_tablet)
response = requests.delete(url_tablet)
if (response.status_code == 200):
master_const.table_names["tables"].remove(dict_return["table_name"])
master_const.server_load_dict[tablet_name] -= 1
master_const.server_table_dict[tablet_name].remove(dict_return["table_name"])
del master_const.table_info[dict_return["table_name"]]
self._set_response(response.status_code)
return
# CLose the lock
elif dict_return["function_name"] == master_const.del_function_types[1]:
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names["tables"]:
self._set_response(404)
return
if dict_return["table_name"] in master_const.locks and client_data["client_id"] not in master_const.locks[
dict_return["table_name"]]:
self._set_response(400)
return
master_const.locks[dict_return["table_name"]].remove(client_data["client_id"])
self._set_response(200)
def do_DELETE(self):
parser_post_type_obj = UrlParser('delete')
dict_return = parser_post_type_obj.parse(self.path)
if dict_return["function_name"] == const.del_function_types[0]:
if dict_return["table_name"] in const.manifest["table_names"][
"tables"]:
const.manifest["table_names"]["tables"].remove(
dict_return["table_name"])
del const.manifest["table_meta_data"][
dict_return["table_name"]]
with open(const.manifest_filename, 'wb') as outfile:
pickle.dump(const.manifest, outfile)
self._set_response(200)
else:
self._set_response(404)
def do_GET(self):
content_length = self.headers['content-length']
get_data = {}
if content_length != None:
content_length = int(content_length)
get_data = self.rfile.read(content_length)
# data = None
parser_get_type_obj = UrlParser('get')
dict_return = parser_get_type_obj.parse(self.path)
# perform on the dict_return
# # Table Get Info
# elif dict_return["function_name"] == const.get_function_types[1]:
# table_info = {}
# if "table_names" in const.manifest:
# if dict_return["table_name"] in const.manifest["table_names"]["tables"]:
# table_info = const.manifest["table_meta_data"][dict_return["table_name"]]
# data_json = json.dumps(table_info)
# self._set_response(200)
# self.wfile.write(data_json.encode("utf8"))
# else:
# self._set_response(404)
# else:
# self._set_response(400)
# retrieve a cell
if dict_return["function_name"] == const.get_function_types[0]:
return_value = const.retrieve(dict_return["table_name"], get_data)
if return_value["success"]:
data_json = json.dumps(return_value["data"])
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
else:
self._set_response(return_value["success_code"])
def do_POST(self):
# example: reading content from HTTP request
data = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# Create a table
if dict_return["function_name"] == const.post_function_types[0]:
my_dict = {}
try:
my_dict = json.loads(post_data)
except:
self._set_response(400)
return
exists = True if "table_meta_data" in const.manifest and my_dict[
'name'] in const.manifest["table_meta_data"] else False
if exists == False:
if "table_names" not in const.manifest:
const.manifest["table_names"] = const.table_names
if "table_meta_data" not in const.manifest:
const.manifest["table_meta_data"] = const.table_meta_data
# Write to manifest variable
const.manifest["table_names"]["tables"].append(my_dict['name'])
const.manifest["table_meta_data"][my_dict['name']] = my_dict
#Write manifest to disk for recovery purposes
with open(const.manifest_filename, 'wb') as outfile:
pickle.dump(const.manifest, outfile)
# const.table_names["tables"].append(my_dict['name'])
# const.table_meta_data[my_dict['name']] = my_dict
#const.mem_table[my_dict['name']] = []
self._set_response(200)
else:
self._set_response(409)
# Insert cell
elif dict_return["function_name"] == const.post_function_types[1]:
self._set_response(
const.operation.insert(dict_return["table_name"], post_data,
False))
# Set max entries
elif dict_return["function_name"] == const.post_function_types[2]:
my_dict = {}
try:
my_dict = json.loads(post_data)
except:
self._set_response(400)
return
if 'memtable_max' not in my_dict:
return 400
new_entries = my_dict['memtable_max']
if not isinstance(new_entries, int):
return 400
const.manifest["max_entries"] = new_entries
with open(const.manifest_filename, 'wb') as outfile:
pickle.dump(const.manifest, outfile)
self._set_response(const.operation.set_max_entries(post_data))
else:
self._set_response(404)
def do_GET(self):
content_length = self.headers['content-length']
get_data = {}
if content_length != None:
content_length = int(content_length)
get_data = self.rfile.read(content_length)
# data = None
parser_get_type_obj = UrlParser('get')
dict_return = parser_get_type_obj.parse(self.path)
# perform on the dict_return
# List Tables
if dict_return["function_name"] == const.get_function_types[0]:
if "table_names" in const.manifest:
data = const.manifest["table_names"]
else:
data = const.table_names
data_json = json.dumps(data)
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
# Table Get Info
elif dict_return["function_name"] == const.get_function_types[1]:
table_info = {}
if "table_names" in const.manifest:
if dict_return["table_name"] in const.manifest["table_names"][
"tables"]:
table_info = const.manifest["table_meta_data"][
dict_return["table_name"]]
data_json = json.dumps(table_info)
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
else:
self._set_response(404)
else:
self._set_response(400)
# retrieve a cell
elif dict_return["function_name"] == const.get_function_types[2]:
return_value = const.operation.retrieve(dict_return["table_name"],
get_data)
if return_value["success"]:
data_json = json.dumps(return_value["data"])
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
else:
self._set_response(return_value["success_code"])
# retrieve cells
elif dict_return["function_name"] == const.get_function_types[3]:
return_value = const.operation.retrieve_cells(
dict_return["table_name"], get_data)
if return_value["success"]:
data_json = json.dumps(return_value["data"])
self._set_response(200)
self.wfile.write(data_json.encode("utf8"))
else:
self._set_response(return_value["success_code"])
# retrieve a row
elif dict_return["function_name"] == const.get_function_types[4]:
# It seems that we don't need to implement it
pass
else:
self._set_response(409)
def do_POST(self):
# example: reading content from HTTP request
if not master_const.check_start:
master_const.check_start = True
try:
x = threading.Thread(target=check_tablets)
x.start()
except KeyboardInterrupt:
pass
except:
print("Error: unable to start thread")
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# Create a table
# Picks the least loaded tablet server
if dict_return["function_name"] == master_const.post_function_types[0]:
if table_dict["name"] in master_const.table_names["tables"]:
self._set_response(409)
return
tablet_name = master_const.master_operation.load_balance()
url_tablet = MasterSupport.url(
master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"], "/Create/")
hostandport = {}
hostandport["hostname"] = master_const.tablets_info[tablet_name][
"hostname"]
hostandport["port"] = master_const.tablets_info[tablet_name][
"port"]
response = requests.post(url_tablet, json=table_dict)
if (response.status_code == 200):
master_const.table_names["tables"].append(table_dict["name"])
master_const.server_load_dict[tablet_name] += 1
master_const.server_table_dict[tablet_name].append(
table_dict["name"])
master_const.table_info[table_dict["name"]] = {}
master_const.table_info[
table_dict["name"]]["name"] = table_dict["name"]
master_const.table_info[table_dict["name"]]["tablets"] = []
master_const.table_info[table_dict["name"]]["tablets"].append(
hostandport)
self._set_response(response.status_code)
# insert cells
# elif dict_return["function_name"] == master_const.post_function_types[1]:
#
# if table_dict["name"] in master_const.table_names["tables"]:
# self._set_response(409)
# return
#
# tablet_name = master_const.master_operation.load_balance()
# url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
# master_const.tablets_info[tablet_name]["port"], "/Create/")
# hostandport = {}
#
# hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
# hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
# response = requests.post(url_tablet, json=table_dict)
#
# self._set_response(response.status_code)
# Open a lock to table
if dict_return["function_name"] == master_const.post_function_types[3]:
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names[
"tables"]:
self._set_response(404)
return
if dict_return["table_name"] not in master_const.locks:
master_const.locks[dict_return["table_name"]] = []
if client_data["client_id"] in master_const.locks[
dict_return["table_name"]]:
self._set_response(400)
else:
master_const.locks[dict_return["table_name"]].append(
client_data["client_id"])
self._set_response(200)
# tablet start
elif dict_return["function_name"] == master_const.post_function_types[
4]:
client_data = json.loads(post_data)
tablet_hostname = client_data["hostname"]
tablet_port = client_data["port"]
tablet_name = "tablet" + str(len(master_const.running_tablets) + 1)
master_const.running_tablets.append(tablet_name)
master_const.tablets_info.update({
tablet_name: {
"hostname": tablet_hostname,
"port": tablet_port
}
})
self._set_response(200)
# sharding
elif dict_return["function_name"] == master_const.post_function_types[
5]:
client_data = json.loads(post_data)
table_name = client_data["table_name"]
row_from_1 = client_data["row_from_1"]
row_to_1 = client_data["row_to_1"]
row_from_2 = client_data["row_from_2"]
row_to_2 = client_data["row_to_2"]
tablet_hostname = client_data["hostname"]
tablet_port = client_data["port"]
new_tablet_name = ""
new_tablet_hostname = ""
new_tablet_port = ""
for running_tablet_name in master_const.running_tablets:
if not master_const.tablets_info[running_tablet_name][
"hostname"] == tablet_hostname or not master_const.tablets_info[
running_tablet_name]["port"] == tablet_port:
new_tablet_name = running_tablet_name
new_tablet_hostname = master_const.tablets_info[
running_tablet_name]["hostname"]
new_tablet_port = master_const.tablets_info[
running_tablet_name]["port"]
break
for running_tablet_name in master_const.running_tablets:
if master_const.tablets_info[running_tablet_name][
"hostname"] == tablet_hostname and master_const.tablets_info[
running_tablet_name]["port"] == tablet_port:
tablet_name = running_tablet_name
if new_tablet_name == "":
# no other tablets
self._set_response(400)
return
# assume we only shard once
if len(master_const.table_info[table_name]["tablets"]) > 1:
self._set_response(401)
return
master_const.table_info[table_name]["tablets"][0].update({
"row_from":
row_from_1,
"row_to":
row_to_1
})
master_const.table_info[table_name]["tablets"].append({
"hostname":
new_tablet_hostname,
"port":
new_tablet_port,
"row_from":
row_from_2,
"row_to":
row_to_2
})
url = MasterSupport.url(new_tablet_hostname, new_tablet_port,
"/Sharding/")
requests.post(url, json=client_data["data"])
self._set_response(200)
# forward
elif dict_return["function_name"] == master_const.post_function_types[
6]:
client_data = json.loads(post_data)
table_name = client_data["table_name"]
table_info = master_const.table_info[table_name]["tablets"][1]
if client_data["data"]["row"] > table_info["row_to"]:
table_info["row_to"] = client_data["data"]["row"]
url = MasterSupport.url(new_tablet_hostname, new_tablet_port,
"/api/table/" + table_name + "/cell")
requests.post(url, json=client_data["data"])
self._set_response(200)
def do_POST(self):
# example: reading content from HTTP request
print("here")
print(self.path)
data = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
# Create a table
path = self.path.split('/')
if path[1] == const.post_function_types[0]:
print("create table")
my_dict = {}
try:
my_dict = json.loads(post_data)
except:
self._set_response(400)
return
exists = True if "table_meta_data" in const.manifest and my_dict[
'name'] in const.manifest["table_meta_data"] else False
if exists == False:
if "table_names" not in const.manifest:
const.manifest["table_names"] = const.table_names
if "table_meta_data" not in const.manifest:
const.manifest["table_meta_data"] = const.table_meta_data
# Write to manifest variable
const.manifest["table_names"]["tables"].append(my_dict['name'])
const.shard_info["table_name"] = my_dict['name']
const.manifest["table_meta_data"][my_dict['name']] = my_dict
# Write manifest to disk for recovery purposes
with open(const.manifest_filename, 'wb') as outfile:
pickle.dump(const.manifest, outfile)
# const.table_names["tables"].append(my_dict['name'])
# const.table_meta_data[my_dict['name']] = my_dict
# const.mem_table[my_dict['name']] = []
self._set_response(200)
else:
self._set_response(409)
print("409")
# Recover
elif path[1] == const.post_function_types[2]:
print("recovering...")
my_dict = {}
try:
my_dict = json.loads(post_data)
except:
self._set_response(400)
return
print(my_dict)
WAL_filename = "WAL_" + my_dict["hostname"] + "_" + str(
my_dict["port"]) + ".txt"
try:
with open(WAL_filename, 'rb') as new_file:
WAL = pickle.load(new_file)
for i in range(len(WAL)):
if table_name not in self.manifest["table_names"][
"tables"]:
const.manifest["table_names"]["tables"].append(
table_name)
const.manifest["table_meta_data"][table_name] = {}
data_json = json.dumps(WAL[i]["cell"])
post_data = data_json.encode("utf8")
const.insert(const.WAL[i]["table_name"], post_data, True)
with open(const.WAL_filename, 'wb') as outfile:
pickle.dump(const.WAL, outfile)
except IOError:
print("WAL file doesn't exist")
pass
manifest_filename = "manifest_" + my_dict["hostname"] + "_" + str(
my_dict["port"]) + ".txt"
try:
with open(manifest_filename, 'rb') as new_file:
manifest = pickle.load(new_file)
# update
for table_name in manifest["ssindex"]:
if table_name not in const.manifest["ssindex"]:
const.manifest["ssindex"].update({table_name: {}})
for column_family in manifest["ssindex"][table_name]:
if column_family not in const.manifest["ssindex"][
table_name]:
const.manifest["ssindex"][table_name].update(
{column_family: {}})
for column in manifest["ssindex"][table_name][
column_family]:
if column not in const.manifest["ssindex"][
table_name][column_family]:
const.manifest["ssindex"][table_name][
column_family].update(
{column: OOBTree()})
for row in manifest["ssindex"][table_name][
column_family][column]:
if row not in const.manifest["ssindex"][
table_name][column_family][column]:
const.manifest["ssindex"][table_name][
column_family][column].update(
{row: []})
const.manifest["ssindex"][table_name][column_family][column][row] += \
manifest["ssindex"][table_name][column_family][column][row]
for table_name in manifest["ssindex_2"]:
if table_name not in const.manifest["ssindex_2"]:
const.manifest["ssindex_2"].update(
{table_name: {}})
for column_family in manifest["ssindex_2"][table_name]:
if column_family not in const.manifest[
"ssindex_2"][table_name]:
const.manifest["ssindex_2"][table_name].update(
{column_family: {}})
for column in manifest["ssindex_2"][table_name][
column_family]:
if column not in const.manifest["ssindex_2"][
table_name][column_family]:
const.manifest["ssindex_2"][table_name][
column_family].update(
{column: OOBTree()})
for row in manifest["ssindex_2"][table_name][
column_family][column]:
if row not in const.manifest["ssindex_2"][
table_name][column_family][column]:
const.manifest["ssindex_2"][
table_name][column_family][
column].update({row: []})
const.manifest["ssindex_2"][table_name][column_family][column][row] += \
manifest["ssindex_2"][table_name][column_family][column][row]
for table_name in manifest["table_names"]["tables"]:
if table_name not in const.manifest["table_names"][
"tables"]:
const.manifest["table_names"]["tables"].append(
table_name)
for table_name in manifest["table_meta_data"]:
if table_name not in const.manifest["table_meta_data"]:
const.manifest["table_meta_data"][table_name] = {
"name": table_name,
"column_families": []
}
const.manifest["table_meta_data"][table_name]["column_families"] += \
manifest["table_meta_data"][table_name]["column_families"]
with open(const.manifest_filename, 'wb') as outfile:
pickle.dump(const.manifest, outfile)
except IOError:
print("manifest file doesn't exist")
pass
self._set_response(200)
# regular check
elif path[1] == const.post_function_types[3]:
self._set_response(200)
else:
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# insert cell
if dict_return["function_name"] == const.post_function_types[1]:
print("insert cell")
self._set_response(
const.insert(dict_return["table_name"], post_data, False))
else:
print(dict_return["function_name"])
self._set_response(409)
def do_POST(self):
# example: reading content from HTTP request
if not master_const.check_start:
master_const.check_start = True
try:
x = threading.Thread(target=check_tablets)
x.start()
except KeyboardInterrupt:
pass
except:
print("Error: unable to start thread")
data = None
table_dict = None
content_length = self.headers['content-length']
if content_length != None:
content_length = int(content_length)
post_data = self.rfile.read(content_length)
try:
table_dict = json.loads(post_data)
except:
self._set_response(400)
return
parser_post_type_obj = UrlParser('post')
dict_return = parser_post_type_obj.parse(self.path)
# Create a table
# Picks the least loaded tablet server
if dict_return["function_name"] == master_const.post_function_types[0]:
if table_dict["name"] in master_const.table_names["tables"]:
self._set_response(409)
return
tablet_name = master_const.master_operation.load_balance()
url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
master_const.tablets_info[tablet_name]["port"], "/Create/")
hostandport = {}
hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
response = requests.post(url_tablet, json=table_dict)
if (response.status_code == 200):
master_const.table_names["tables"].append(table_dict["name"])
master_const.server_load_dict[tablet_name] += 1
master_const.server_table_dict[tablet_name].append(table_dict["name"])
master_const.table_info[table_dict["name"]] = {}
master_const.table_info[table_dict["name"]]["name"] = table_dict["name"]
master_const.table_info[table_dict["name"]]["tablets"] = []
master_const.table_info[table_dict["name"]]["tablets"].append(hostandport)
self._set_response(response.status_code)
# insert cells
# elif dict_return["function_name"] == master_const.post_function_types[1]:
#
# if table_dict["name"] in master_const.table_names["tables"]:
# self._set_response(409)
# return
#
# tablet_name = master_const.master_operation.load_balance()
# url_tablet = MasterSupport.url(master_const.tablets_info[tablet_name]["hostname"],
# master_const.tablets_info[tablet_name]["port"], "/Create/")
# hostandport = {}
#
# hostandport["hostname"] = master_const.tablets_info[tablet_name]["hostname"]
# hostandport["port"] = master_const.tablets_info[tablet_name]["port"]
# response = requests.post(url_tablet, json=table_dict)
#
# self._set_response(response.status_code)
# Open a lock to table
if dict_return["function_name"] == master_const.post_function_types[3]:
client_data = json.loads(post_data)
if dict_return["table_name"] not in master_const.table_names["tables"]:
self._set_response(404)
return
if dict_return["table_name"] not in master_const.locks:
master_const.locks[dict_return["table_name"]] = []
if client_data["client_id"] in master_const.locks[dict_return["table_name"]]:
self._set_response(400)
else:
master_const.locks[dict_return["table_name"]].append(client_data["client_id"])
self._set_response(200)