def update(self): # update inner data self.replica_map = {} self.replication_task = [] # update content_set, replica_map for server in self.server_set: file_list = util.get_file_list_on_server(server) for file_uuid in file_list: self.content_set.add(file_uuid) if file_uuid not in self.replica_map: self.replica_map[file_uuid] = [] if server not in self.replica_map[file_uuid]: self.replica_map[file_uuid].append(server) current_timestamp = int(time.time()) logs = self.aggregator.get_redirect_log_entries( self.last_timestamp, current_timestamp) # used recently generated redirect logs to instruct replication for log in logs: timestamp, uuid, source, source_uuid, dest, req_type, status, response_size = log dest = util.convert_to_local_hostname(dest) if (uuid, dest) not in self.replication_task: self.replication_task.append((uuid, dest)) self.last_timestamp = current_timestamp
def read_file(uuid, source_uuid, source, dest, delay): print 'READ: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid + ', delay: ' + str( delay) query_parameters = {'uuid': uuid, 'ip': source, 'source_uuid': source_uuid} if delay is not None: query_parameters['delay'] = delay dest = util.convert_to_local_hostname(dest) read_url = 'http://%s/read?%s' % (dest, urllib.urlencode(query_parameters)) print read_url # may need get latency number r = requests.get(read_url, stream=True) if (r.status_code == requests.codes.ok): with open(CLIENT_DOWNLOAD_FOLDER + uuid, 'wb') as fd: chunk_size = 1024 for chunk in r.iter_content(chunk_size): fd.write(chunk) fd.close() print 'DONE: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid return True # request failed print 'FAIL: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid print '\tSTATUS: ' + str(r.status_code) + ', TEXT: ' + r.text return False
def total_server_capacity(self, server): server = util.convert_to_local_hostname(server) url = 'http://%s/capacity?%s' % (server, urllib.urlencode({'file_size': 0})) r = requests.get(url, timeout=30) return float(r.text)
def migrate_to_locations(self, placements_by_server): for optimal_server, uuids in placements_by_server.iteritems(): for uuid in uuids: current_server = self.uuid_metadata[uuid]['current_server'] # conver to local hostname in case of simulation current_server = util.convert_to_local_hostname(current_server) optimal_server = util.convert_to_local_hostname(optimal_server) if current_server != optimal_server: url = 'http://%s/transfer?%s' % ( current_server, urllib.urlencode({ 'uuid': uuid, 'destination': optimal_server })) print url r = requests.put(url, timeout=30) if r.status_code == requests.codes.ok: print 'SUCCESS: Migrating ' + uuid + ' from <' + current_server + '> to <' + optimal_server + '>' else: raise Exception('FAILED: Migrating ' + uuid + ' from <' + current_server + '> to <' + optimal_server + '>')
def write_file(uuid, source, dest, response_size): print 'WRITE: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size query_parameters = {'uuid': uuid, 'ip': source} dest = util.convert_to_local_hostname(dest) write_url = 'http://%s/write?%s' % (dest, urllib.urlencode(query_parameters)) print write_url # make the content of the file the file's theoretical size files = {'file': response_size} r = requests.post(write_url, files=files) if (r.status_code != requests.codes.created): print 'FAIL: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size return None print 'DONE: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size return uuid
def collapse_to_datacenters(self, locations_by_uuid): placements_by_server = {} for server in self.servers: placements_by_server[server] = [] for uuid, location in locations_by_uuid.iteritems(): metadata = { 'current_server': None, 'optimal_location': location, 'uuid': uuid, 'dist': None, 'file_size': None, 'request_count': None } best_server = None # Query any server for metadata - server will update and get information any_server = util.convert_to_local_hostname(self.servers[0]) url = 'http://%s/metadata?%s' % (any_server, urllib.urlencode({'uuid': uuid})) print url r = requests.get(url) print r.text response = json.loads(r.text) metadata['current_server'] = response['server'] metadata['file_size'] = response['file_size'] metadata['request_count'] = self.log_manager.successful_read_count( uuid) best_servers = self.find_closest_servers(location) best_server = best_servers[0] metadata['dist'] = best_server['distance'] self.uuid_metadata[uuid] = metadata placements_by_server[best_server['server']].append(uuid) placements_by_server = self.redistribute_server_data_by_capacity( placements_by_server) return placements_by_server
def distributed_replication(filename, ip_address, delay_time, metadata): concurrent_requests = metadata.get_concurrent_request(filename) if concurrent_requests is not None: # Make sure that the number of concurrent requests is under k. # If not, replicate to another server. if int(concurrent_requests) >= int(app.config['k']): # 1) Find the closest server. known_servers = metadata.get_all_server(app.config['HOST']) concurrent_connections = metadata.get_concurrent_connections( filename) closest_servers = dict() for concurrent_connection in concurrent_connections: closest_server = util.find_closest_servers_with_ip( concurrent_connection, known_servers)[0] if closest_server['server'] not in closest_servers: closest_servers[closest_server['server']] = 1 else: closest_servers[closest_server['server']] += 1 # target_server = max(closest_servers) target_server = max(closest_servers.iteritems(), key=operator.itemgetter(1))[0] target_server = util.convert_to_local_hostname(target_server) # 2) Check if there is enough space on the remote server. url = 'http://%s/can_move_file?%s' % (target_server, urllib.urlencode( { 'uuid': filename, 'file_size': 0, 'delay': delay_time })) response = requests.get(url) if response.status_code == requests.codes.ok: # 3) Copy the file to that server. clone_file(request.args.get('uuid'), target_server, 'DISTRIBUTED_REPLICATE', ip_address) else: raise Exception( 'Something fishy is going on... Should have at least one request')