def map_node(node_addr):
if node_addr.startswith('ERROR:'):
raise KeyError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
map_msg = 'map\n'
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
proc_id = node_connect.receive()
if proc_id.startswith('ERROR:'):
raise RuntimeError(proc_id)
for help_file in helping_files:
with Connect(node_host, node_port) as node_connect:
help_file_local_path = help_file.split('/')[-1]
file_add_msg = line_packing(
'file_add', proc_id, help_file_local_path)
node_connect.send(file_add_msg + '\n')
file_send_start_status = node_connect.receive()
if file_send_start_status == 'ok':
node_connect.send_file(proc_id, help_file)
map_msg = line_packing(
'map_run', table_in, table_out, script, proc_id, line_packing(helping_files))
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
node_connect.receive()
def write(self, table_path):
sent = False
line = False
while not sent:
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send('write')
node_addr = connect.receive()
if not node_addr:
raise RuntimeError('An error has occured')
if node_addr.startswith('ERROR:'):
raise OutOfRecourseError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
with Connect(node_host, node_port) as node_connect:
node_connect.send('write\n')
node_connect.send(table_path + '\n')
received = 'ok'
while True:
if not line:
line = sys.stdin.readline()
print 'read line:', line
if line == '':
sent = True
break
node_connect.send(line)
received = node_connect.receive()
if received == 'ok':
line = False
else:
break
def delete(self, table_path):
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send_once('delete\n' + table_path)
nodes_with_table = connect.receive_by_line()
if not nodes_with_table:
raise RuntimeError('Cannot get nodes with table')
for node_addr in nodes_with_table:
if node_addr.startswith('ERROR:'):
raise KeyError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
with Connect(node_host, node_port) as node_connect:
node_connect.send('delete\n')
node_connect.send(table_path + '\n')
def delete_table(slave, connect, args):
table_path = next(args)
slave.delete_table(table_path)
with Connect(*master_addr) as master_connect:
removed_table_inform = line_packing('table_remove', slave_addr[0],
slave_addr[1], table_path)
master_connect.send_once(removed_table_inform)
def list_dir(self, cluster_path):
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send(line_packing('list_dir', cluster_path))
cluster_info = connect.receive_by_line()
for cl in cluster_info:
if cl.startswith('ERROR:'):
raise RuntimeError(cl)
yield cl
def get_table_info(self, table_path):
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send('table_info\n')
connect.send(table_path)
table_info = connect.receive()
if table_info.startswith('ERROR:'):
raise KeyError(table_info)
return table_info
def handle_connection(master, conn, addr):
with Connect(socket=conn) as connect:
action = connect.receive()
print 'action:', action
command, args = parse_input(action)
try:
handle_command(command, master, connect, args)
except ServerError as e:
print e.msg
inform_error(connect, e.msg)
def handle_connection(slave, file_manager, conn, addr):
with Connect(socket=conn) as connect:
data_recv = connect.receive_by_line()
command = next(data_recv)
try:
handle_command(command, slave, file_manager, connect, data_recv)
except ServerError as e:
inform_error(connect, e.msg)
except StopIteration as e:
inform_error(connect, 'ERROR: Invalid message')
except Exception as e:
inform_error(connect, str(e))
def map_start(slave, file_manager, connect, args):
table_in = next(args)
table_out = next(args)
script = next(args)
proc_id = next(args)
exec_dir = file_manager.get_slot_path(proc_id)
is_mapped, size_written = slave.map(table_in, table_out, script, exec_dir)
connect.send_once('ok' if is_mapped else 'not ok')
with Connect(*master_addr) as master_connect:
new_table_inform = line_packing('table_add', slave_addr[0],
slave_addr[1], table_out, size_written)
master_connect.send_once(new_table_inform)
def write(slave, connect, args):
table_path = next(args)
size_written = 0
for line in args:
is_written = slave.write_table_line(table_path, line)
if not is_written:
connect.send_once('not ok')
break
else:
connect.send('ok')
size_written += len(line)
with Connect(*master_addr) as master_connect:
new_table_inform = line_packing('table_add', slave_addr[0],
slave_addr[1], table_path,
size_written)
master_connect.send_once(new_table_inform)
def map(self, table_in, table_out, script, helping_files=[]):
is_all_mapped = True
with Connect(self._master_config.host, self._master_config.port) as connect:
connect.send_once(line_packing('map', table_in, table_out))
map_info = connect.receive_by_line()
if not map_info:
raise KeyError('Cannot get nodes with table')
map_id = next(map_info)
if map_id.startswith('ERROR:'):
raise KeyError(map_id)
def map_node(node_addr):
if node_addr.startswith('ERROR:'):
raise KeyError(node_addr)
node_host, node_port = node_addr.split(':')
node_port = int(node_port)
map_msg = 'map\n'
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
proc_id = node_connect.receive()
if proc_id.startswith('ERROR:'):
raise RuntimeError(proc_id)
for help_file in helping_files:
with Connect(node_host, node_port) as node_connect:
help_file_local_path = help_file.split('/')[-1]
file_add_msg = line_packing(
'file_add', proc_id, help_file_local_path)
node_connect.send(file_add_msg + '\n')
file_send_start_status = node_connect.receive()
if file_send_start_status == 'ok':
node_connect.send_file(proc_id, help_file)
map_msg = line_packing(
'map_run', table_in, table_out, script, proc_id, line_packing(helping_files))
with Connect(node_host, node_port) as node_connect:
node_connect.send_once(map_msg)
node_connect.receive()
for node_addr in map_info:
thread = Thread(target=map_node, args=(node_addr,))
thread.start()
return map_id
def connect_to_master(size_limit):
connect_msg = line_packing('new_slave', slave_addr[0], slave_addr[1],
size_limit)
Connect(*master_addr).send_once(connect_msg)