def __init__(self, master_sock, chunk_id, map_fn, combine_fn):
ProtocolThread.__init__(self, is_server=False)
self.master_sock = master_sock
self.socks.append(self.master_sock)
self.chunk_id = chunk_id
self.map_fn = map_fn
self.combine_fn = combine_fn
def __init__(self, client_sock, path, path_results, map_mod, combine_mod,
reduce_mod):
ProtocolThread.__init__(self, is_server=False)
self.client_sock = client_sock
self.socks.append(self.client_sock)
self.path = path
self.path_results = path_results
self.map_mod = map_mod
self.combine_mod = combine_mod
self.reduce_mod = reduce_mod
self.reduce_fn = deserialize_module(self.reduce_mod).reduce_fn
self.counts = []
self.chunks = dict(fs.get_file_chunks(self.path))
self.current_assignments = {
follower.ip_addr: None
for follower in followers
}
self.map_chunks_assigned = []
self.map_chunks_completed = []
self.map_result_chunks = {
follower.ip_addr: []
for follower in followers
}
self.map_key_values = defaultdict(lambda: [])
self.outstanding_reduce_followers = []
self.followers_with_map = {}
self.add_command('map_response', self.handle_map_response)
self.add_command('reduce_response', self.handle_reduce_response)
def __init__(self, client_sock, path, path_results, map_mod, combine_mod, reduce_mod):
ProtocolThread.__init__(self, is_server=False)
self.client_sock = client_sock
self.socks.append(self.client_sock)
self.path = path
self.path_results = path_results
self.map_mod = map_mod
self.combine_mod = combine_mod
self.reduce_mod = reduce_mod
self.reduce_fn = deserialize_module(self.reduce_mod).reduce_fn
self.counts = []
self.chunks = dict(fs.get_file_chunks(self.path))
self.current_assignments = {follower.ip_addr: None for follower in followers}
self.map_chunks_assigned = []
self.map_chunks_completed = []
self.map_result_chunks = {follower.ip_addr: [] for follower in followers}
self.map_key_values = defaultdict(lambda: [])
self.outstanding_reduce_followers = []
self.followers_with_map = {}
self.add_command('map_response', self.handle_map_response)
self.add_command('reduce_response', self.handle_reduce_response)
def __init__(self, master_sock, result_chunk_id, map_chunk_ids, reduce_fn):
ProtocolThread.__init__(self, is_server=False)
self.master_sock = master_sock
self.socks.append(self.master_sock)
self.result_chunk_id = result_chunk_id
self.map_chunk_ids = map_chunk_ids
self.reduce_fn = reduce_fn
def __init__(self):
ProtocolThread.__init__(self, 'localhost', loc.master_client_port, is_server=True)
self.manipulators = {}
self.add_command('ls', self.handle_ls)
self.add_command('rm', self.handle_rm)
self.add_command('mkdir', self.handle_mkdir)
self.add_command('cat', self.handle_cat)
self.add_command('upload_chunk', self.handle_upload_chunk)
self.add_command('map_reduce', self.handle_map_reduce_upload)
def __init__(self, client_sock):
ProtocolThread.__init__(self, is_server=False)
self.client_sock = client_sock
if self.client_sock:
self.socks.append(self.client_sock)
self.expected_results = 0
self.results = 0
self.add_command('store_chunk', self.handle_store_chunk)
self.add_command('remove_chunk', self.handle_remove_chunk)
self.add_command('get_chunk', self.handle_get_chunk)
def __init__(self, client_sock):
ProtocolThread.__init__(self, is_server=False)
self.client_sock = client_sock
if self.client_sock:
self.socks.append(self.client_sock)
self.expected_results = 0
self.results = 0
self.add_command('store_chunk', self.handle_store_chunk)
self.add_command('remove_chunk', self.handle_remove_chunk)
self.add_command('get_chunk', self.handle_get_chunk)
def __init__(self, master_ip):
ProtocolThread.__init__(self, 'localhost', loc.follower_port, is_server=True)
master_sock = self.add_socket(master_ip, loc.master_follower_port)
master_sock.send(own_ip_address)
self.remove_socket(master_sock)
self.add_command('store_chunk', self.handle_store_chunk)
self.add_command('remove_chunk', self.handle_remove_chunk)
self.add_command('get_chunk', self.handle_get_chunk)
self.add_command('map', self.handle_map)
self.add_command('reduce', self.handle_reduce)
def __init__(self):
ProtocolThread.__init__(self,
'localhost',
loc.master_client_port,
is_server=True)
self.manipulators = {}
self.add_command('ls', self.handle_ls)
self.add_command('rm', self.handle_rm)
self.add_command('mkdir', self.handle_mkdir)
self.add_command('cat', self.handle_cat)
self.add_command('upload_chunk', self.handle_upload_chunk)
self.add_command('map_reduce', self.handle_map_reduce_upload)
def __init__(self, master_ip):
ProtocolThread.__init__(self, is_server=False)
self.master_ip = master_ip
self.sock = self.add_socket(master_ip, loc.master_client_port)
self.complete = False
self.expected_results = 1
self.results = 0
self.data_source = None
self.lines_per_chunk = 0
self.path = None
self.add_command('ls', self.handle_result)
self.add_command('rm', self.handle_result)
self.add_command('mkdir', self.handle_result)
self.add_command('get_chunk', self.handle_result)
self.add_command('upload_chunk', self.handle_upload)
self.add_command('remove_chunk', self.handle_result)
self.add_command('map_reduce', self.handle_result)
def __init__(self, master_ip):
ProtocolThread.__init__(self, is_server=False)
self.master_ip = master_ip
self.sock = self.add_socket(master_ip, loc.master_client_port)
self.complete = False
self.expected_results = 1
self.results = 0
self.data_source = None
self.lines_per_chunk = 0
self.path = None
self.add_command('ls', self.handle_result)
self.add_command('rm', self.handle_result)
self.add_command('mkdir', self.handle_result)
self.add_command('get_chunk', self.handle_result)
self.add_command('upload_chunk', self.handle_upload)
self.add_command('remove_chunk', self.handle_result)
self.add_command('map_reduce', self.handle_result)
def __init__(self):
ProtocolThread.__init__(self,
'localhost',
loc.master_follower_port,
is_server=True)
def __init__(self):
ProtocolThread.__init__(self, 'localhost', loc.master_follower_port, is_server=True)