def update(self):
# update inner data
self.replica_map = {}
self.replication_task = []
# update content_set, replica_map
for server in self.server_set:
file_list = util.get_file_list_on_server(server)
for file_uuid in file_list:
self.content_set.add(file_uuid)
if file_uuid not in self.replica_map:
self.replica_map[file_uuid] = []
if server not in self.replica_map[file_uuid]:
self.replica_map[file_uuid].append(server)
current_timestamp = int(time.time())
logs = self.aggregator.get_redirect_log_entries(
self.last_timestamp, current_timestamp)
# used recently generated redirect logs to instruct replication
for log in logs:
timestamp, uuid, source, source_uuid, dest, req_type, status, response_size = log
dest = util.convert_to_local_hostname(dest)
if (uuid, dest) not in self.replication_task:
self.replication_task.append((uuid, dest))
self.last_timestamp = current_timestamp
def read_file(uuid, source_uuid, source, dest, delay):
print 'READ: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid + ', delay: ' + str(
delay)
query_parameters = {'uuid': uuid, 'ip': source, 'source_uuid': source_uuid}
if delay is not None:
query_parameters['delay'] = delay
dest = util.convert_to_local_hostname(dest)
read_url = 'http://%s/read?%s' % (dest, urllib.urlencode(query_parameters))
print read_url
# may need get latency number
r = requests.get(read_url, stream=True)
if (r.status_code == requests.codes.ok):
with open(CLIENT_DOWNLOAD_FOLDER + uuid, 'wb') as fd:
chunk_size = 1024
for chunk in r.iter_content(chunk_size):
fd.write(chunk)
fd.close()
print 'DONE: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid
return True
# request failed
print 'FAIL: source: ' + source + ', uuid: ' + uuid + ', source_uuid: ' + source_uuid
print '\tSTATUS: ' + str(r.status_code) + ', TEXT: ' + r.text
return False
def total_server_capacity(self, server):
server = util.convert_to_local_hostname(server)
url = 'http://%s/capacity?%s' % (server,
urllib.urlencode({'file_size': 0}))
r = requests.get(url, timeout=30)
return float(r.text)
def migrate_to_locations(self, placements_by_server):
for optimal_server, uuids in placements_by_server.iteritems():
for uuid in uuids:
current_server = self.uuid_metadata[uuid]['current_server']
# conver to local hostname in case of simulation
current_server = util.convert_to_local_hostname(current_server)
optimal_server = util.convert_to_local_hostname(optimal_server)
if current_server != optimal_server:
url = 'http://%s/transfer?%s' % (
current_server,
urllib.urlencode({
'uuid': uuid,
'destination': optimal_server
}))
print url
r = requests.put(url, timeout=30)
if r.status_code == requests.codes.ok:
print 'SUCCESS: Migrating ' + uuid + ' from <' + current_server + '> to <' + optimal_server + '>'
else:
raise Exception('FAILED: Migrating ' + uuid +
' from <' + current_server + '> to <' +
optimal_server + '>')
def write_file(uuid, source, dest, response_size):
print 'WRITE: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size
query_parameters = {'uuid': uuid, 'ip': source}
dest = util.convert_to_local_hostname(dest)
write_url = 'http://%s/write?%s' % (dest,
urllib.urlencode(query_parameters))
print write_url
# make the content of the file the file's theoretical size
files = {'file': response_size}
r = requests.post(write_url, files=files)
if (r.status_code != requests.codes.created):
print 'FAIL: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size
return None
print 'DONE: source: ' + source + ', uuid: ' + uuid + ', response_size: ' + response_size
return uuid
def collapse_to_datacenters(self, locations_by_uuid):
placements_by_server = {}
for server in self.servers:
placements_by_server[server] = []
for uuid, location in locations_by_uuid.iteritems():
metadata = {
'current_server': None,
'optimal_location': location,
'uuid': uuid,
'dist': None,
'file_size': None,
'request_count': None
}
best_server = None
# Query any server for metadata - server will update and get information
any_server = util.convert_to_local_hostname(self.servers[0])
url = 'http://%s/metadata?%s' % (any_server,
urllib.urlencode({'uuid': uuid}))
print url
r = requests.get(url)
print r.text
response = json.loads(r.text)
metadata['current_server'] = response['server']
metadata['file_size'] = response['file_size']
metadata['request_count'] = self.log_manager.successful_read_count(
uuid)
best_servers = self.find_closest_servers(location)
best_server = best_servers[0]
metadata['dist'] = best_server['distance']
self.uuid_metadata[uuid] = metadata
placements_by_server[best_server['server']].append(uuid)
placements_by_server = self.redistribute_server_data_by_capacity(
placements_by_server)
return placements_by_server
def distributed_replication(filename, ip_address, delay_time, metadata):
concurrent_requests = metadata.get_concurrent_request(filename)
if concurrent_requests is not None:
# Make sure that the number of concurrent requests is under k.
# If not, replicate to another server.
if int(concurrent_requests) >= int(app.config['k']):
# 1) Find the closest server.
known_servers = metadata.get_all_server(app.config['HOST'])
concurrent_connections = metadata.get_concurrent_connections(
filename)
closest_servers = dict()
for concurrent_connection in concurrent_connections:
closest_server = util.find_closest_servers_with_ip(
concurrent_connection, known_servers)[0]
if closest_server['server'] not in closest_servers:
closest_servers[closest_server['server']] = 1
else:
closest_servers[closest_server['server']] += 1
# target_server = max(closest_servers)
target_server = max(closest_servers.iteritems(),
key=operator.itemgetter(1))[0]
target_server = util.convert_to_local_hostname(target_server)
# 2) Check if there is enough space on the remote server.
url = 'http://%s/can_move_file?%s' % (target_server,
urllib.urlencode(
{
'uuid': filename,
'file_size': 0,
'delay': delay_time
}))
response = requests.get(url)
if response.status_code == requests.codes.ok:
# 3) Copy the file to that server.
clone_file(request.args.get('uuid'), target_server,
'DISTRIBUTED_REPLICATE', ip_address)
else:
raise Exception(
'Something fishy is going on... Should have at least one request')