Esempio n. 1
0
 def __init__(self, master_sock, chunk_id, map_fn, combine_fn):
     ProtocolThread.__init__(self, is_server=False)
     self.master_sock = master_sock
     self.socks.append(self.master_sock)
     self.chunk_id = chunk_id
     self.map_fn = map_fn
     self.combine_fn = combine_fn
Esempio n. 2
0
    def __init__(self, client_sock, path, path_results, map_mod, combine_mod,
                 reduce_mod):
        ProtocolThread.__init__(self, is_server=False)
        self.client_sock = client_sock
        self.socks.append(self.client_sock)
        self.path = path
        self.path_results = path_results
        self.map_mod = map_mod
        self.combine_mod = combine_mod
        self.reduce_mod = reduce_mod
        self.reduce_fn = deserialize_module(self.reduce_mod).reduce_fn
        self.counts = []

        self.chunks = dict(fs.get_file_chunks(self.path))
        self.current_assignments = {
            follower.ip_addr: None
            for follower in followers
        }

        self.map_chunks_assigned = []
        self.map_chunks_completed = []
        self.map_result_chunks = {
            follower.ip_addr: []
            for follower in followers
        }
        self.map_key_values = defaultdict(lambda: [])

        self.outstanding_reduce_followers = []
        self.followers_with_map = {}

        self.add_command('map_response', self.handle_map_response)
        self.add_command('reduce_response', self.handle_reduce_response)
Esempio n. 3
0
    def __init__(self, client_sock, path, path_results, map_mod, combine_mod, reduce_mod):
        ProtocolThread.__init__(self, is_server=False)
        self.client_sock = client_sock
        self.socks.append(self.client_sock)
        self.path = path
        self.path_results = path_results
        self.map_mod = map_mod
        self.combine_mod = combine_mod
        self.reduce_mod = reduce_mod
        self.reduce_fn = deserialize_module(self.reduce_mod).reduce_fn
        self.counts = []

        self.chunks = dict(fs.get_file_chunks(self.path))
        self.current_assignments = {follower.ip_addr: None for follower in followers}

        self.map_chunks_assigned = []
        self.map_chunks_completed = []
        self.map_result_chunks = {follower.ip_addr: [] for follower in followers}
        self.map_key_values = defaultdict(lambda: [])

        self.outstanding_reduce_followers = []
        self.followers_with_map = {}

        self.add_command('map_response', self.handle_map_response)
        self.add_command('reduce_response', self.handle_reduce_response)
Esempio n. 4
0
 def __init__(self, master_sock, result_chunk_id, map_chunk_ids, reduce_fn):
     ProtocolThread.__init__(self, is_server=False)
     self.master_sock = master_sock
     self.socks.append(self.master_sock)
     self.result_chunk_id = result_chunk_id
     self.map_chunk_ids = map_chunk_ids
     self.reduce_fn = reduce_fn
Esempio n. 5
0
 def __init__(self):
     ProtocolThread.__init__(self, 'localhost', loc.master_client_port, is_server=True)
     self.manipulators = {}
     self.add_command('ls', self.handle_ls)
     self.add_command('rm', self.handle_rm)
     self.add_command('mkdir', self.handle_mkdir)
     self.add_command('cat', self.handle_cat)
     self.add_command('upload_chunk', self.handle_upload_chunk)
     self.add_command('map_reduce', self.handle_map_reduce_upload)
Esempio n. 6
0
 def __init__(self, client_sock):
     ProtocolThread.__init__(self, is_server=False)
     self.client_sock = client_sock
     if self.client_sock:
         self.socks.append(self.client_sock)
     self.expected_results = 0
     self.results = 0
     self.add_command('store_chunk', self.handle_store_chunk)
     self.add_command('remove_chunk', self.handle_remove_chunk)
     self.add_command('get_chunk', self.handle_get_chunk)
Esempio n. 7
0
 def __init__(self, client_sock):
     ProtocolThread.__init__(self, is_server=False)
     self.client_sock = client_sock
     if self.client_sock:
         self.socks.append(self.client_sock)
     self.expected_results = 0
     self.results = 0
     self.add_command('store_chunk', self.handle_store_chunk)
     self.add_command('remove_chunk', self.handle_remove_chunk)
     self.add_command('get_chunk', self.handle_get_chunk)
Esempio n. 8
0
 def __init__(self, master_ip):
     ProtocolThread.__init__(self, 'localhost', loc.follower_port, is_server=True)
     master_sock = self.add_socket(master_ip, loc.master_follower_port)
     master_sock.send(own_ip_address)
     self.remove_socket(master_sock)
     self.add_command('store_chunk', self.handle_store_chunk)
     self.add_command('remove_chunk', self.handle_remove_chunk)
     self.add_command('get_chunk', self.handle_get_chunk)
     self.add_command('map', self.handle_map)
     self.add_command('reduce',  self.handle_reduce)
Esempio n. 9
0
 def __init__(self):
     ProtocolThread.__init__(self,
                             'localhost',
                             loc.master_client_port,
                             is_server=True)
     self.manipulators = {}
     self.add_command('ls', self.handle_ls)
     self.add_command('rm', self.handle_rm)
     self.add_command('mkdir', self.handle_mkdir)
     self.add_command('cat', self.handle_cat)
     self.add_command('upload_chunk', self.handle_upload_chunk)
     self.add_command('map_reduce', self.handle_map_reduce_upload)
Esempio n. 10
0
    def __init__(self, master_ip):
        ProtocolThread.__init__(self, is_server=False)
        self.master_ip = master_ip
        self.sock = self.add_socket(master_ip, loc.master_client_port)
        self.complete = False
        self.expected_results = 1
        self.results = 0

        self.data_source = None
        self.lines_per_chunk = 0
        self.path = None

        self.add_command('ls', self.handle_result)
        self.add_command('rm', self.handle_result)
        self.add_command('mkdir', self.handle_result)
        self.add_command('get_chunk', self.handle_result)
        self.add_command('upload_chunk', self.handle_upload)
        self.add_command('remove_chunk', self.handle_result)
        self.add_command('map_reduce', self.handle_result)
Esempio n. 11
0
    def __init__(self, master_ip):
        ProtocolThread.__init__(self, is_server=False)
        self.master_ip = master_ip
        self.sock = self.add_socket(master_ip, loc.master_client_port)
        self.complete = False
        self.expected_results = 1
        self.results = 0

        self.data_source = None
        self.lines_per_chunk = 0
        self.path = None

        self.add_command('ls', self.handle_result)
        self.add_command('rm', self.handle_result)
        self.add_command('mkdir', self.handle_result)
        self.add_command('get_chunk', self.handle_result)
        self.add_command('upload_chunk', self.handle_upload)
        self.add_command('remove_chunk', self.handle_result)
        self.add_command('map_reduce', self.handle_result)
Esempio n. 12
0
 def __init__(self):
     ProtocolThread.__init__(self,
                             'localhost',
                             loc.master_follower_port,
                             is_server=True)
Esempio n. 13
0
 def __init__(self):
     ProtocolThread.__init__(self, 'localhost', loc.master_follower_port, is_server=True)