def execute_task(self, socket, message):
if self.current_task_cap == 0:
print("Worker: Ups! NO capacity for more tasks")
answer = mt.Message().set_message('ERROR', {
'info': "Worker busy",
'type_error': 'WorkerBusyError'
})
socket.send_string(str(answer))
return None
print("Worker: ", 'Vamos a ejecutar un task')
self.current_task_cap -= 1
arguments = message.payload
current_ip = self.current_worker_addr[0]
task_ping_addr = (current_ip, mt.get_available_random_port(current_ip))
task_executor = TaskTracker(self.filesystem_broadcast_addr,
task_ping_addr, self.current_worker_addr,
**arguments)
f = lambda task_ex: task_ex.execute_task()
procces = Process(target=f,
args=(task_executor, ),
name='worker_task_proccess')
print('Worker: ', "submited task")
procces.start()
answer = mt.Message().set_message('OK', {
'task_ping_ip': task_ping_addr[0],
'task_ping_port': task_ping_addr[1]
})
socket.send_string(str(answer))
print('Worker: ', "This is my current task capacity: ",
self.current_task_cap)
def try_to_get_rows(self, socket, message):
new_port = mt.get_available_random_port(self.addr[0])
if not os.path.exists(message.payload["status_db_url"]):
socket.send_string(
str(Message().set_message(
'ERROR', {
'info': 'unable to open database file',
'type_error': 'FileNotFoundError'
})))
new_addr = (self.addr[0], new_port)
try:
database = sql.connect(message.payload["status_db_url"])
except sql.OperationalError:
socket.send_string(
str(Message().set_message(
'ERROR', {
'info': 'unable to open database file',
'type_error': "OperationalError"
})))
return -1
socket.send_string(
str(Message().set_message("OK", {
"ip": self.addr[0],
"port": new_port
})))
p = Process(target=self.new_get_rows, args=(
new_addr,
message,
))
p.daemon = True
p.start()
def execute_job(self, socket, message):
if self.current_job_cap == 0:
print("Worker: Ups! NO capacity for more tasks")
answer = mt.Message().set_message('ERROR', {
'info': "Worker busy",
'type_error': 'WorkerBusyError'
})
socket.send_string(str(answer))
return None
self.current_job_cap -= 1
arguments = message.payload
tracker_ip = self.current_worker_addr[0]
tracker_addr_ping = (tracker_ip, mt.get_available_random_port())
tracker = JobTracker(self.worker_broadcast_addr,
self.filesystem_broadcast_addr, tracker_addr_ping,
tracker_ip, self.current_worker_addr, **arguments)
f = lambda track: track.execute_job()
procces = Process(target=f,
args=(tracker, ),
name='worker_procces_job')
procces.start()
print('Worker: ', "submited job")
answer = mt.Message().set_message(
'OK', {
"ping_tracker_ip": tracker_addr_ping[0],
"ping_tracker_port": tracker_addr_ping[1]
})
socket.send_string(str(answer))
print('Worker: ', "This is my current job capacity: ",
self.current_job_cap)
def try_to_put_job(self, socket, message):
print("Intentemos hacer putjob")
job_id = message.payload["job_id"]
# files = message.payload["files"]
db_url = str(job_id) + "/database.sqlite3"
# print("files to save:", files)
try:
os.mkdir(job_id)
except FileExistsError:
print(
"FS:Ya esa tarea existe, escribiré sobre la carpeta ya creada")
new_port = mt.get_available_random_port(self.addr[0])
message_to_send = Message().set_message("OK", {
"ip": self.addr[0],
"port": new_port
})
socket.send_string(str(message_to_send))
new_addr = (self.addr[0], new_port)
p = Process(target=self.new_put_job, args=(
new_addr,
job_id,
db_url,
))
p.daemon = True
p.start()
def try_to_put_data(self, socket, message):
print("FS: Enter PutData")
new_port = mt.get_available_random_port(self.addr[0])
is_a_byte_message = message.payload["byte"]
filename = message.payload["file_url"]
lock, err = self._check_file_lock(filename)
if lock is None:
print("FS: Error 403, en el putdata")
socket.send_string(
str(Message().set_message(
"Error", {
"info": "Someone is using the file. Cannot access.",
'type_error': err
})))
return -1
print("FS: Creado el lock haremos put_data")
try:
if is_a_byte_message:
# data_message = data_socket.recv()
file = open(filename, "wb")
else:
# data_message = Message().get_message(data_socket.recv_string())
file = open(filename, "w")
print("FS:File correctly open")
m = Message().set_message("OK", {
"ip": self.addr[0],
"port": new_port
})
socket.send_string(str(m))
print("FS:OK message sent")
except zmq.error.ZMQError:
socket.send_string(
str(Message().set_message(
"ERROR",
{"info": "Something went wrong creating the file."})))
os.remove(lock)
print("FS: Removemos el lock: ", lock)
return -1
new_addr = (self.addr[0], new_port)
p = Process(target=self.new_put_data,
args=(
new_addr,
file,
is_a_byte_message,
filename,
lock,
))
p.daemon = True
p.start()
def try_to_insert_job(self, socket, message):
new_port = mt.get_available_random_port(self.addr[0])
socket.send_string(
str(Message().set_message("OK", {
"ip": self.addr[0],
"port": new_port
})))
new_addr = (self.addr[0], new_port)
p = Process(target=self.new_insert_job, args=(
new_addr,
message,
))
p.daemon = True
p.start()
def __init__(self,
data="./data",
functions_file="./function.py",
data_type='text',
client_ip='0.0.0.0',
workers=[]):
"""
The initializer of the Client instance.
:param data:
The url of the file where the data to be processed is stored on the client
:param functions_file:
The url of the file where the file (*.py) with the map and reduce functions is stored on the client
:param data_type:
The type of the data stored on the data file, might be 'text' or 'table'.
Is 'text' in the case of needing to be interpreted as plain text
Is 'table' if the data is well organized to be interpreted as a table
:param timeout:
The timeout for the client to wait for an answer from the service manager
"""
self.tries = 5
self.functions_file = functions_file
self.data_type = data_type
self.job_url = None
self.data = data
self.master_ping_addr = tuple()
self.master_ping_context = zmq.Context()
self.master_ping_socket = None
self.client_ip = client_ip
self.listener_addr = ("0.0.0.0", "7999")
self.filesystem_addr = None
self.filesystem_addrs = []
self.status_handler = None
self.masters_list = workers
self.current_master = None
self._get_new_master()
self.files_urls = {}
self.jhash = None
# Si solamente recibimos str como data
extra = self._get_extra_info_from_job(data, functions_file)
self.job_info_for_register = extra
self.listener_addr = (self.client_ip,
mt.get_available_random_port(self.client_ip))
def _load_send_task_phase(self):
'''
En la fase de send task el status es que tenemos insertado los bloques y estan almenos en estado ADDED
y a lo sumo en estado SUBMITTED. Lo que haremos será cargar todos los bloques que estan en la fase actual(map o
reduce), los que estén en ADDED los mandamos con sendtaskmessage, entonces en este punto tenemos todos en
almenos SUBMITTED y empezamos el ciclo en wait_result.
:return:
'''
print("JobTracker: ", 'Tratamos de cargar la fase de sendtask')
filters = [('phase', self.job_phase)]
blocks, blocks_status = self._get_blocks_urls(filters)
print(
"JobTracker: Tratando de cargar la fase de sendtask queda al cargar los bloques:"
"------->blocks ", blocks, '---> status ', blocks_status)
unsubmitted_blocks = [
block for i, block in enumerate(blocks) if
blocks_status[i]['state'] == mt.slices_states[0] and len(block) > 0
]
print("JobTracker: Unsubmittedblocks: ", unsubmitted_blocks)
workers_addrs, workers_status = self._getting_workers()
put_byte = self._check_if_data_is_byte()
assign_tasks = {
block_row['block_id']:
(block_row['worker_ip'], block_row['worker_port'])
for i, block_row in enumerate(blocks_status)
if block_row['worker_ip'] != ''
}
# si quedó algún bloque sin asignar
if len(unsubmitted_blocks) > 0:
_, assign_tasks2 = self.distribute_slices(unsubmitted_blocks,
workers_status, put_byte,
True)
for x, y in assign_tasks2.items():
assign_tasks[x] = y
else:
r_port = mt.get_available_random_port(self.tracker_ip)
self.tracker_addr = (self.tracker_ip, r_port)
# Ahora en este estado todos los bloques estan en estado de SUBMITTED
print("JobTracker: ", "Pasamos a la fase de wait_results")
self.wait_results(assign_tasks, workers_addrs, put_byte)
def try_to_insert_blockresult(self, socket, message):
new_port = mt.get_available_random_port(self.addr[0])
if not os.path.exists(message.payload["status_db_url"]):
socket.send_string(
str(Message().set_message(
'ERROR', {
'info': 'unable to open database file',
'type_error': 'FileNotFoundError'
})))
socket.send_string(
str(Message().set_message("OK", {
"ip": self.addr[0],
"port": new_port
})))
new_addr = (self.addr[0], new_port)
p = Process(target=self.new_insert_blockresult,
args=(
new_addr,
message,
))
p.daemon = True
p.start()
def distribute_slices(self,
blocks,
workers_status,
put_byte=False,
registered=False):
'''
Manda los tasks messages y devuelve la tupla de slices_urls y assign_tasks
:param blocks:
:param workers_status:
:param put_byte:
:return:
'''
r_port = mt.get_available_random_port(self.tracker_ip)
self.tracker_addr = (self.tracker_ip, r_port)
assign_tasks = self.send_tasks_messages(blocks, put_byte,
workers_status, registered)
return blocks, assign_tasks
def wait_for_results(self, assign_tasks, workers_addrs, put_byte):
context = zmqa.Context()
answer_socket = context.socket(zmq.REP)
try:
answer_socket.bind('tcp://{}:{}'.format(*self.tracker_addr))
except zmq.error.ZMQError:
print("JobTracker: ZMQERROR in wait_for_results")
answer_socket.bind('tcp://{}:{}'.format(
self.tracker_ip,
mt.get_available_random_port(self.tracker_ip)))
print('JobTracker: ',
"Ahora el tracker espera por respuestas DONE por: ",
self.tracker_addr)
print("JobTracker: ", 'Esperando Done de: ', workers_addrs)
cnt_answers = 0
while True:
ans = mt.loop_tool(self._try_to_recv_done, answer_socket)
if ans is None:
submitted_filters = [('state', mt.slices_states[1]),
('phase', self.job_phase)]
writing_filters = [('state', mt.slices_states[-2]),
('phase', self.job_phase)]
print(
"JobTracker: Buscamos los submitted y los writing blocks")
submitted_blocks_rows = self.status_handler.get_status_rows(
self.file_sys_addr, 'block', submitted_filters)
writing_blocks_rows = self.status_handler.get_status_rows(
self.file_sys_addr, 'block', writing_filters)
print("JobTracker: ",
"Ya recibimos los bloques de submitted y writing")
if len(submitted_blocks_rows) == 0:
if len(writing_blocks_rows) == 0:
print("JobTracker: ",
'Ya todos los bloques estan en DONE')
return 0
continue
testing_blocks = self._get_blocks_urls_with_blocks_rows(
submitted_blocks_rows)
if testing_blocks == 0:
answer_socket.close()
return 0
# print("JobTracker: ",'ESTOS SON LOS BLOQUES QUE QUEDAN ES ESTADO SUBMITTED: ',testing_blocks)
indexes_to_remove = []
for i, block in enumerate(testing_blocks):
block_id = submitted_blocks_rows[i]['block_id']
worker_assigned_ping = (
submitted_blocks_rows[i]['worker_ip'],
submitted_blocks_rows[i]['worker_port'])
# Todo: tengo que revisar porqué me dan key error si no se han caido ningun worker
print("JobTracker: ",
"ESTE ES EL WORKER ASSIGN DE BLOCK_ID ", block_id,
" worker: ", worker_assigned_ping)
print("JobTracker: ", 'Vamos a hacerle ping a :',
worker_assigned_ping, ' con addr ping: ',
worker_assigned_ping)
answer = mt.loop_tool(
mt.try_to_send_message, mt.send_message,
mt.Message().set_message("PING"), worker_assigned_ping,
lambda: print("JobTracker: ", 'No me respondio: ',
worker_assigned_ping), 1)
if answer is not None:
print(
"JobTracker: Ya se que este worker sigue pinchando: ",
worker_assigned_ping)
indexes_to_remove.append(i)
else:
print("JobTracker: Parece que murio este worker: ",
worker_assigned_ping)
print("JobTracker: ", 'buscamos otro worker')
print("JobTracker: ",
'Ya le hicimo ping a todos, pacemos a reasignar')
testing_blocks = [
testing_blocks[i] for i in range(len(testing_blocks))
if i not in indexes_to_remove
]
print(
"JobTracker: ",
'ESTOS SON LOS BLOQUES QUE QUEDAN(MOD) En ESTADO SUBMITTED: ',
testing_blocks, ' QUE NO ME RESPONDEN')
if len(testing_blocks) == 0:
continue
workers_addrs, workers_status = self._getting_workers()
assign_tasks2 = self.send_tasks_messages(testing_blocks,
put_byte,
workers_status,
registered_block=True)
for x, y in assign_tasks2.items():
assign_tasks[x] = y
continue
answer = mt.Message().get_message(ans)
if answer.message_name == 'ERROR':
# self.send_error_message_to_addr(answer.payload,self.client_addr)
print("JobTracker: ", "Error por parte del worker: ",
answer.payload['info'])
elif answer.message_name == "DONE":
worker_addr = answer.payload['worker_addr']
worker_addr = (worker_addr[0], worker_addr[1])
print('JobTracker: ',
"Recibi un Done de: {}:{}".format(*worker_addr))
message = mt.Message().set_message("OK")
print("JobTracker: ", "Sending OK to: ", worker_addr)
answer_socket.send_string(str(message))
cnt_answers += 1
def try_to_get_data(self, socket, message):
print("FS: Tratamos de hacer getdata")
new_port = mt.get_available_random_port(self.addr[0])
file_url = message.payload["file_url"]
if not os.path.exists(file_url):
print("FS: No existe: ", file_url)
message_to_send = Message().set_message(
"ERROR", {
"info":
"File does not exist: {}. "
"I can't access this file right now.".format(file_url),
'type_error':
'FileNotFound'
})
socket.send_string(str(message_to_send))
# assert False,"este fue el mensaje que me habian mandado: {}".format(str(message))
return -1
print("FS: Revisemos que nadie mas lo esta usando")
lock, err = self._check_file_lock(message.payload["file_url"])
if lock is None:
print(
"FS: Error 403.1: Temporarily Forbidden. I can't access this file right now."
)
message_to_send = Message().set_message(
"ERROR", {
"info": "Temporarily Forbidden. "
"I can't access this file right now.",
'type_error': err
})
socket.send_string(str(message_to_send))
return -1
try:
if message.payload["byte"]:
data = open("./" + message.payload["file_url"], "rb")
else:
data = open("./" + message.payload["file_url"], "r")
message_to_send = Message().set_message("OK", {
"ip": str(self.addr[0]),
"port": new_port
})
socket.send_string(str(message_to_send))
except FileNotFoundError:
message_to_send = Message().set_message(
"ERROR", {
"info": "File is not in the filesystem}",
"type_error": 'FileNotFound'
})
socket.send_string(str(message_to_send))
# data.close()
os.remove(lock)
print("FS: Removemos el lock: ", lock)
# assert False, "este fue el mensaje que me habian mandado: {}".format(str(message))
return -1
new_addr = (self.addr[0], new_port)
p = Process(target=self.new_get_data,
args=(
new_addr,
data,
lock,
message,
))
p.daemon = True
p.start()
