def map_node(node_addr):
if node_addr.startswith('ERROR:'):
raise KeyError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
map_msg = 'map\n'
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
proc_id = node_connect.receive()
if proc_id.startswith('ERROR:'):
raise RuntimeError(proc_id)
for help_file in helping_files:
with Connect(node_host, node_port) as node_connect:
help_file_local_path = help_file.split('/')[-1]
file_add_msg = line_packing(
'file_add', proc_id, help_file_local_path)
node_connect.send(file_add_msg + '\n')
file_send_start_status = node_connect.receive()
if file_send_start_status == 'ok':
node_connect.send_file(proc_id, help_file)
map_msg = line_packing(
'map_run', table_in, table_out, script, proc_id, line_packing(helping_files))
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
node_connect.receive()
def delete_table(slave, connect, args):
table_path = next(args)
slave.delete_table(table_path)
with Connect(*master_addr) as master_connect:
removed_table_inform = line_packing('table_remove', slave_addr[0],
slave_addr[1], table_path)
master_connect.send_once(removed_table_inform)
def list_dir(self, cluster_path):
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send(line_packing('list_dir', cluster_path))
cluster_info = connect.receive_by_line()
for cl in cluster_info:
if cl.startswith('ERROR:'):
raise RuntimeError(cl)
yield cl
def list_dir(master, connect, args):
if len(args) != 1:
raise IncorrectMessageError()
cluster_path = args[0]
cluster_info = master.get_cluster_info(cluster_path)
if not cluster_info:
raise ServerError('Cluster not found')
connect.send_once(line_packing(*cluster_info))
def map_request(master, connect, args):
if len(args) != 2:
raise IncorrectMessageError()
table_in, table_out = args
map_info = mapper.create_map(table_in, table_out)
table_nodes = master.get_table_nodes(table_in)
table_nodes = map(lambda node: node.url, table_nodes)
connect.send_once(line_packing(map_info.id, *table_nodes))
def delete_table_request(master, connect, args):
if len(args) != 1:
raise IncorrectMessageError()
table_path = args[0]
table_info = master.get_table_info(table_path)
if table_info is None:
raise ServerError('Requested table does not exist')
table_nodes = table_info.nodes
nodes_addr = map(lambda node: node.url, table_nodes)
connect.send_once(line_packing(*nodes_addr))
def map_start(slave, file_manager, connect, args):
table_in = next(args)
table_out = next(args)
script = next(args)
proc_id = next(args)
exec_dir = file_manager.get_slot_path(proc_id)
is_mapped, size_written = slave.map(table_in, table_out, script, exec_dir)
connect.send_once('ok' if is_mapped else 'not ok')
with Connect(*master_addr) as master_connect:
new_table_inform = line_packing('table_add', slave_addr[0],
slave_addr[1], table_out, size_written)
master_connect.send_once(new_table_inform)
def write(slave, connect, args):
table_path = next(args)
size_written = 0
for line in args:
is_written = slave.write_table_line(table_path, line)
if not is_written:
connect.send_once('not ok')
break
else:
connect.send('ok')
size_written += len(line)
with Connect(*master_addr) as master_connect:
new_table_inform = line_packing('table_add', slave_addr[0],
slave_addr[1], table_path,
size_written)
master_connect.send_once(new_table_inform)
def map(self, table_in, table_out, script, helping_files=[]):
is_all_mapped = True
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send_once(line_packing('map', table_in, table_out))
map_info = connect.receive_by_line()
if not map_info:
raise KeyError('Cannot get nodes with table')
map_id = next(map_info)
if map_id.startswith('ERROR:'):
raise KeyError(map_id)
def map_node(node_addr):
if node_addr.startswith('ERROR:'):
raise KeyError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
map_msg = 'map\n'
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
proc_id = node_connect.receive()
if proc_id.startswith('ERROR:'):
raise RuntimeError(proc_id)
for help_file in helping_files:
with Connect(node_host, node_port) as node_connect:
help_file_local_path = help_file.split('/')[-1]
file_add_msg = line_packing(
'file_add', proc_id, help_file_local_path)
node_connect.send(file_add_msg + '\n')
file_send_start_status = node_connect.receive()
if file_send_start_status == 'ok':
node_connect.send_file(proc_id, help_file)
map_msg = line_packing(
'map_run', table_in, table_out, script, proc_id, line_packing(helping_files))
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
node_connect.receive()
for node_addr in map_info:
thread = Thread(target=map_node, args=(node_addr,))
thread.start()
return map_id
def connect_to_master(size_limit):
connect_msg = line_packing('new_slave', slave_addr[0], slave_addr[1],
size_limit)
Connect(*master_addr).send_once(connect_msg)