def __init__(self):
self.port_ip = uwsgi.opt["http"].split(":")
self.data_path =\
"/opt/gfdenis/metadata/" + self.port_ip[0] + "/" + self.port_ip[1]
self.storage_nodes = []
self.proxy_nodes = []
kwargs = {'node_id': uwsgi.opt['node_id'],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': uwsgi.opt["node_type"]}
self.group = GroupMessaging(kwargs=kwargs)
self.group.start()
kwargs = {'node_id': "PROXY_" + self.port_ip[0] + "_" + self.port_ip[1],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': "NAMENODESTORAGE"}
self.group_proxy_storage = GroupMessaging(kwargs=kwargs)
self.group_proxy_storage.start()
self.http = uwsgi.opt['http']
self.node_id = uwsgi.opt['node_id']
logging.basicConfig(filename="/var/log/denisfs/proxy_" +\
self.node_id+ "_" + self.http + '.log',\
level=logging.DEBUG)
class Application(ApplicationBase):
def __init__(self):
self.port_ip = uwsgi.opt["http"].split(":")
self.data_path = "/opt/gfdenis/data/" +\
self.port_ip[0] + "/" + self.port_ip[1] + "/"
self.node_id = uwsgi.opt['node_id']
self.http = uwsgi.opt['http']
logging.basicConfig(filename="/var/log/denisfs/storage_" +\
self.node_id + "_" + self.http + '.log',\
level=logging.DEBUG)
kwargs = {'node_id': uwsgi.opt['node_id'],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': uwsgi.opt["node_type"]}
self.group = GroupMessaging(kwargs=kwargs)
self.group.start()
kwargs = {'node_id': "DATA_" + self.port_ip[0] + "_" + self.port_ip[1],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': "NAMENODESTORAGE"}
self.group_proxy_storage = GroupMessaging(kwargs=kwargs)
self.group_proxy_storage.start()
def _unique_file_dir(self, filename):
"""
Generate a unique path for a filename
"""
return self.data_path + filename + "/"
def _post_request(self, filename, host, headers=None):
response = {"headers": [], "status": "200 OK", "body": ""}
try:
r = requests.post("http://"+host[0] + ":"+ host[1] +
"/" + filename, headers=headers)
response['status'] = \
utils.http_code[str(r.status_code)]
response['body'] = str(r.content)
except requests.ConnectionError:
response['status'] = utils.http_code['500']
response['body'] = "No file created. Unknow error"
return response
def PUT(self, request):
"""
Create path if needed and write data to it
"""
logging.debug("storage.put")
#request.showVariables()
response = {"headers": [], "status": "201 OK", "body": ""}
if not request.file_name:
response['status'] = http_code['304']
return response
replication = request.metadata.get("HTTP_REPLICATION", None)
if replication == "1":
logging.debug("REPLICATION")
fm = FileManager(self.data_path + "/" + request.file_name,\
request.file_name)
if not fm.write_file(request.body):
response['status'] = http_code['500']
response['body'] = "Could not replicate"
logging.debug("Could not replicate")
return response
client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
logging.debug("HTTP_CLIENT_ID: %s" % client_id)
replicas_count = request.metadata.get("HTTP_REPLICAS_COUNT", 3)
fm = FileManager(self.data_path + "/" + request.file_name\
, request.file_name)
if not fm.write_file(request.body):
response['status'] = http_code['500']
logging.debug("Starting to save data in disk")
storage_replicas = make_tuple(request.metadata["HTTP_STORAGE_LIST"])
for replica in storage_replicas:
if replica[0] == self.port_ip[0] and\
replica[1] == self.port_ip[1]:
continue
headers = {}
headers['file_name'] = request.file_name
headers['data_length'] = request.content_length
headers['etag'] = request.file_name
headers['replicas_count'] = replicas_count
headers['client_id'] = "%s:%s" % (request.remote_addr,
request.remote_port)
headers["replication"] = "1"
response = self._put_request(request.file_name,
replica,
data=request.body,
headers=headers)
logging.debug("response: %s" % response)
if not "201" in response["status"]:
response['status'] = http_code['500']
response['body'] = "Could reach proper number of replicas"
return response
logging.debug("Starting to send metadata confirmation for proxy")
proxy_list = make_tuple(request.metadata["HTTP_PROXY_LIST"])
logging.debug(proxy_list)
for proxy in proxy_list:
logging.debug(proxy)
headers = {}
headers['data_length'] = request.content_length
headers['etag'] = request.file_name
headers['replicas_count'] = 3
headers['client_id'] = client_id
response = self._post_request(request.file_name, proxy,
headers=headers)
logging.debug("response: %s" % response)
if "201" in response["status"]:
break
return response
def GET(self, request):
"""
Read data from file
"""
logging.debug("storage.get")
#request.showVariables()
response = {"headers": [], "status": "200 OK", "body": ""}
if not request.file_name:
response['status'] = http_code['304']
return response
path = self._unique_file_dir(request.file_name)
#try:
#path = self._unique_file_dir(request.file_name)
#logging.debug("path: %s" % path)
#with open(path + "tombstone", "rb") as f:
#response['status'] = http_code['404']
#response['body'] = "File in tombstone mode"
#return response
#except Exception as e:
#print(e)
try:
with open(path + request.file_name, "rb") as f:
response['body'] = f.read()
except Exception as e:
print(e)
response['status'] = http_code['500']
#logging.debug("response: %s" % response["body"])
return response
def DELETE(self, request):
"""
Create a tombstone file to register the file as deleted
"""
print(request.showVariables())
response = {"headers": [], "status": "200 OK", "body": ""}
try:
path = self._unique_file_dir(request.file_name)
if not os.path.exists(path):
os.makedirs(path)
with open(path + "tombstone", "wb") as f:
f.write(request.body)
except Exception as e:
print(e)
response['status'] = http_code['500']
return response
class Application(ApplicationBase):
# WSGI Server Application designed to act as a distributed metadata manager.
# Consistency is guarantee by election a leader to play a central self
# for mutual exclusion role.
def __init__(self):
self.port_ip = uwsgi.opt["http"].split(":")
self.data_path =\
"/opt/gfdenis/metadata/" + self.port_ip[0] + "/" + self.port_ip[1]
self.storage_nodes = []
self.proxy_nodes = []
kwargs = {'node_id': uwsgi.opt['node_id'],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': uwsgi.opt["node_type"]}
self.group = GroupMessaging(kwargs=kwargs)
self.group.start()
kwargs = {'node_id': "PROXY_" + self.port_ip[0] + "_" + self.port_ip[1],
'address': (uwsgi.opt["broker_ip"], uwsgi.opt['broker_port']),
'node_type': "NAMENODESTORAGE"}
self.group_proxy_storage = GroupMessaging(kwargs=kwargs)
self.group_proxy_storage.start()
self.http = uwsgi.opt['http']
self.node_id = uwsgi.opt['node_id']
logging.basicConfig(filename="/var/log/denisfs/proxy_" +\
self.node_id+ "_" + self.http + '.log',\
level=logging.DEBUG)
def get_nodes(self, proxy_nodes, storage_nodes):
for key, value in self.group_proxy_storage.workers.iteritems():
#print("key: %s" % key)
#print("value: %s" % value)
for node_type in value:
temp_list = node_type.split("_")
#print("temp_list: %s" % temp_list)
if "DATA" in temp_list[1]:
#print("value: %s" % value)
ip = temp_list[2]
port = temp_list[3]
storage_nodes.append((ip, port))
if "PROXY" in temp_list[1]:
#print("value: %s" % value)
ip = temp_list[2]
port = temp_list[3]
proxy_nodes.append((ip, port))
# It validates the file and a possible storage;
# It saves the file_data entry in the proxy datebase
#
# Return to the client the storage node chosen for that file
def PUT(self, request):
logging.debug("proxy.put")
proxy_nodes = []
storage_nodes = []
self.get_nodes(proxy_nodes, storage_nodes)
print("storage_node: %s" % storage_nodes)
print("proxy_node: %s" % proxy_nodes)
#request.showVariables()
print("Am I leader? %s" % self.group.is_leader)
response = {"headers": [], "status": "200 OK", "body": ""}
replication = request.metadata.get("HTTP_REPLICATION", None)
if replication == "1":
logging.debug("REPLICATION")
fm = FileManager(self.data_path + "/" + request.file_name,\
request.file_name)
if not fm.write_file(request.body):
response['status'] = http_code['500']
response['body'] = "Could not replicate"
logging.debug("Could not replicate")
return response
if len(proxy_nodes) < 3:
response['status'] = http_code['304']
response['body'] = "Deny of service due server is down"
print("Deny of service due server is down")
logging.debug("Deny of service due server is down")
return response
if not self.group.is_leader:
response['status'] = http_code['304']
response['body'] = "No leader. Try to find the leader"
logging.debug("I'm no leader. Try to find the leader")
return response
if not request.file_name:
response['status'] = http_code['400']
response['body'] = "No valid file name found"
logging.debug("No valid file name found")
return response
client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
#logging.debug("HTTP_CLIENT_ID: %s" % client_id)
metadata = MetadataManager(self.data_path + "/" + request.file_name,
request.file_name)
file_exist = metadata.restore()
if len(metadata.lock_list) > 0:
response['status'] = http_code['403']
response['body'] = "File is locked"
logging.debug("file is locked")
return response
metadata.size = request.metadata["HTTP_DATA_LENGTH"]
metadata.etag = request.metadata["HTTP_ETAG"]
# Create File in any nodes
if not file_exist:
write_balancer = WriteLoadBalancer(self.data_path, storage_nodes)
balanced_storage_list = write_balancer.balance()
if balanced_storage_list:
storage_list = balanced_storage_list
else:
storage_list = storage_nodes
logging.debug("storage_list: %s" % storage_list)
replicas_count = request.metadata.get("HTTP_REPLICAS_COUNT", 3)
replica_list = storage_list[:int(replicas_count)]
logging.debug("replicas_list: %s" % replica_list)
metadata.storage_list = replica_list
metadata.replicas_count = replicas_count
# Update File in the same nodes
else:
replica_list = metadata.storage_list
available_replicas = []
for replica in metadata.storage_list:
for node in storage_nodes:
if replica[0] == node[0] and replica[1] == node[1]:
available_replicas.append((replica[0], replica[1]))
if len(available_replicas) != len(metadata.storage_list):
response['status'] = http_code['304']
response['body'] = "Deny of service due server is down"
print("Deny of service due server is down")
logging.debug("Deny of service due server is down")
return response
#if len(replica_list) < 3:
#response['status'] = http_code['304']
#response['body'] = "Deny of service due server is down"
#return response
#fl = FileLock(self.data_path + "/" + request.file_name)
#while True:
#if not fl.is_locked():
#break
#else:
#logging.debug("file is locked. Someone is updating")
#time.sleep(1)
#response['status'] = http_code['403']
#response['body'] = "File is locked. Someone is updating."
#return response
# Cuts off not needed replicas
metadata.add_lock_item(client_id, "LOCK_WRITE")
metadata.save()
#fl.lock()
body = {}
body["STORAGE_LIST"] = replica_list
response['body'] = json.dumps(body)
response['status'] = http_code['201']
logging.debug("response: %s" % response)
return response
def POST(self, request):
logging.debug("proxy.post")
proxy_nodes = []
storage_nodes = []
self.get_nodes(proxy_nodes, storage_nodes)
print("storage_node: %s" % storage_nodes)
print("proxy_node: %s" % proxy_nodes)
#request.showVariables()
response = {"headers": [], "status": "201 OK", "body": ""}
if not request.file_name:
response['status'] = http_code['400']
return response
if not self.group.is_leader:
response['status'] = http_code['304']
response['body'] = "No leader. Try to find the leader"
logging.debug("I'm no leader. Try to find the leader")
return response
#client_id = request.metadata.get("HTTP_CLIENT_ID", "test:test")
#logging.debug("HTTP_CLIENT_ID: %s" % client_id)
replicas_count = request.metadata["HTTP_REPLICAS_COUNT"]
metadata = MetadataManager(self.data_path + "/" + request.file_name,
request.file_name,
request.metadata["HTTP_DATA_LENGTH"],
replicas_count,
request.metadata["HTTP_ETAG"])
if not metadata.restore():
response['status'] = http_code['304']
response['body'] = "File does not exist"
logging.debug("File does not exist")
return response
logging.debug(self.data_path + "/" + request.file_name)
fl = FileLock(self.data_path + "/" + request.file_name)
if metadata.lock_list:
metadata.lock_list.pop()
metadata.save()
fl.unlock()
logging.debug("self.proxy_nodes: %s" % self.proxy_nodes)
for proxy_replicas in proxy_nodes:
if proxy_replicas[0] == self.port_ip[0] and\
proxy_replicas[1] == self.port_ip[1]:
continue
headers = {}
headers['file_name'] = request.file_name
headers['data_length'] = request.content_length
headers['etag'] = request.file_name
headers['replicas_count'] = replicas_count
headers['client_id'] = "%s:%s" % (request.remote_addr,
request.remote_port)
headers["replication"] = "1"
metadata_copy = json.dumps(metadata.resource_dict)
response = self._put_request(request.file_name,
proxy_replicas,
data=metadata_copy,
headers=headers)
logging.debug("response: %s" % response)
return response
def GET(self, request):
logging.debug("proxy.get")
response = {"headers": [], "status": "200 OK", "body": ""}
print("Am I leader? %s" % self.group.is_leader)
proxy_nodes = []
storage_nodes = []
self.get_nodes(proxy_nodes, storage_nodes)
print("storage_node: %s" % storage_nodes)
print("proxy_node: %s" % proxy_nodes)
#request.showVariables()
if not request.file_name:
response['status'] = http_code['400']
return response
metadata = MetadataManager(self.data_path + "/" + request.file_name,
request.file_name)
file_exist = metadata.restore()
# File is locked
if metadata.lock_list:
# File is not commited yet
if len(metadata.storage_list) == 0:
response['status'] = http_code['404']
response['body'] = "File is not fully created yet"
print("file is not fully created yet")
return response
if not file_exist:
response['status'] = http_code['404']
response['body'] = "File does not exist"
print("file does not exist")
return response
available_replicas = []
for replica in metadata.storage_list:
for node in storage_nodes:
if replica[0] == node[0] and replica[1] == node[1]:
available_replicas.append((replica[0], replica[1]))
response['body'] = json.dumps(available_replicas)
return response
def DELETE(self, request):
logging.debug("proxy.delete")
response = {"headers": [], "status": "200 OK", "body": ""}
# Deletion happens when leader wants to sync data deletion to
# other nodes
deletion = request.metadata.get("HTTP_DELETION", None)
if deletion == "1":
logging.debug("DELETION")
metadata = MetadataManager(self.data_path + "/" + request.file_name,\
request.file_name)
if not metadata.delete():
response['status'] = http_code['304']
response['body'] = "Could not delete"
logging.debug("Could not delete")
return response
if not self.group.is_leader:
response['status'] = http_code['304']
response['body'] = "No leader. Try to find the leader"
logging.debug("I'm no leader. Try to find the leader")
return response
proxy_nodes = []
storage_nodes = []
self.get_nodes(proxy_nodes, storage_nodes)
print("storage_node: %s" % storage_nodes)
print("proxy_node: %s" % proxy_nodes)
if not request.file_name:
response['status'] = http_code['400']
return response
metadata = MetadataManager(self.data_path + "/" + request.file_name,
request.file_name)
file_exist = metadata.restore()
# File is locked
if metadata.lock_list:
# File is not commited yet
if len(metadata.storage_list) == 0:
response['status'] = http_code['304']
response['body'] = "File is not fully created yet"
print("file is not fully created yet")
logging.debug("file is not fully created yet")
return response
else:
response['status'] = http_code['304']
response['body'] = "File is locked"
print("file is locked")
return response
if not file_exist:
response['status'] = http_code['404']
response['body'] = "File does not exist"
print("file does not exist")
return response
#metadata.delete()
logging.debug("self.proxy_nodes: %s" % proxy_nodes)
if len(proxy_nodes) < 3:
response['status'] = http_code['304']
response['body'] = "Deny of service due server is down"
print("Deny of service due replica down")
logging.debug("Deny of service due replica down")
return response
metadata.delete()
for proxy_replicas in proxy_nodes:
if proxy_replicas[0] == self.port_ip[0] and\
proxy_replicas[1] == self.port_ip[1]:
continue
headers = {}
headers['file_name'] = request.file_name
headers['data_length'] = request.content_length
headers['etag'] = request.file_name
headers['client_id'] = "%s:%s" % (request.remote_addr,
request.remote_port)
headers["deletion"] = "1"
response = self._delete_request(request.file_name,
proxy_replicas,
headers=headers)
logging.debug("response: %s" % response)
return response
