def update_data(self):
"""
Returns:
"""
count = 1
lim = 2
while True:
try:
data = protocol.recv_data(self.sock)
except socket.timeout:
break
data = protocol.parse(data)
if data['type'] == protocol.Type.CONTROL and \
data['subtype'] == protocol.Subtype.CONTROL.stream_end:
if count < lim:
count += 1
else:
break
elif data['type'] == protocol.Type.SNAKE and \
data['subtype'] == protocol.Subtype.SNAKE.new:
snake = game.objects.Snake.create_snake(
game.objects.Point(*data['head']), data['name'],
data['mass'], data['tail'])
self.snakes[data['id']] = snake
if self.key == data['id']:
self.render_control.player = snake
elif data['type'] == protocol.Type.SNAKE and \
data['subtype'] == protocol.Subtype.SNAKE.full_update:
if not data['id'] in self.snakes:
continue
self.snakes[data['id']].update_snake(data['mass'],
data['head'],
data['tail'])
elif data['type'] == protocol.Type.SNAKE and \
data['subtype'] == protocol.Subtype.SNAKE.delete:
del self.snakes[data['id']]
elif data['type'] == protocol.Type.ORB and \
data['subtype'] == protocol.Subtype.ORB.new:
orb = game.objects.Orb(
game.objects.Point(data['x'], data['y']), data['mass'],
data['color'])
self.orbs[data['id']] = orb
elif data['type'] == protocol.Type.ORB and \
data['subtype'] == protocol.Subtype.ORB.delete:
del self.orbs[data['id']]
def _recv(self):
# Get the header first.
try:
header = ''
while len(header) < protocol.HEADER_SIZE:
received = self.socket.recv(protocol.HEADER_SIZE - len(header))
header += received
if len(received) == 0:
return None
except:
return None
size = protocol.packet_length(header)
msg = ""
while len(msg) < size:
received = self.socket.recv(size)
msg += received
if len(received) == 0:
LOGGER.warning("Server socket died while receiving.")
return None
return protocol.parse(header, msg)
def _recv(self):
# Get the header first.
try:
header = ''
while len(header) < protocol.HEADER_SIZE:
received = self.socket.recv(protocol.HEADER_SIZE - len(header))
header += received
if len(received) == 0:
return None
except:
return None
size = protocol.packet_length(header)
msg = ""
while len(msg) < size:
received = self.socket.recv(size)
msg += received
if len(received) == 0:
LOGGER.warning("Server socket died while receiving.")
return None
return protocol.parse(header, msg)
def __init__(self, ip, name):
"""
Args:
ip (str): valid ip of the server.
name (str): the name of the player.
"""
# variables declaration
self.sock = socket.socket()
self.key = None
self.snakes = {}
self.orbs = {}
self.display = pygame.display.get_surface().get_rect()
self.board = None
self.render_control = None
# initial connection
self.sock.connect((ip, protocol.PORT))
self.sock.settimeout(None)
# get server initial
while True:
data = protocol.parse(protocol.recv_data(self.sock))
if data['type'] == protocol.Type.INITIAL and \
data['subtype'] == protocol.Subtype.INITIAL.server:
self.board = pygame.Rect(0, 0, data['width'], data['height'])
self.key = data['id']
break
self.render_control = game.render.Render(self.display, self.board,
self.snakes, self.orbs)
# send client initial
protocol.send_data(self.sock, protocol.initial_client(name))
# end init function and add timeout
self.sock.settimeout(0.001)
def handler(incoming):
cmd = parse(incoming)
return execute(cmd)
def run(self):
"""Execute the receiver thread."""
while self.running:
# repeatedly try to connect to server
while not self.connected:
self.logger.info('waiting for connection')
try:
self.socket.connect((self.server_ip, self.port))
except:
time.sleep(2)
else:
self.connected = True
message = protocol.thread.new_socket(socket=self.socket)
self.network_queue.put(message)
self.logger.info('connected to server')
# loop running while the client is connected
while self.connected:
# if receive fail, go back to reconnection
try:
size_str = self.socket.recv(4)
except:
self.connected = False
if size_str != '':
size = protocol.get_size(size_str)
received = ''
# receive until the whole message is received
while size != 0:
try:
part = self.socket.recv(size)
except:
part = ''
# if the connection is close,
# return to waiting to connection
if part == '':
self.connected = False
break
received += part
size -= len(part)
# if the messaged was received without failing,
# pass it to the logic
if self.connected:
self.logger.debug(
'received message. length: %s' % len(received))
message = protocol.parse(received)
self.logic_queue.put(message)
# if the message type is 'kill', the thread will exit
# this must happen here and not in the logic thread,
# because the logic thread has no way to send a message
# to this thread because it doesn't have a queue.
if message['type'] == 'kill':
self.socket.close()
self.connected = False
self.running = False
break
else:
self.connected = False
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def parse_cmd(self, cmd, size, args, transp):
protocol.parse(cmd, size, args)
def parse_cmd(self, cmd, size, args, transport):
protocol.parse(cmd, size, args, transport)
def run(self):
"""
Execute the restore thread.
The thread first collect all blocks from the clients.
Then, It maps the received blocks to a giant dict.
Example:
{1:
{blocks: {1: ['10.0.0.9/file.dat_1.data',
'10.0.0.10/file.dat_1.data',
'10.0.0.11/file.dat_1.data'],
2: ['10.0.0.9/file.dat_2.data',
'10.0.0.10/file.dat_2.data',
'10.0.0.11/file.dat_2.data'],
3: ['10.0.0.9/file.dat_3.data',
'10.0.0.10/file.dat_3.data',
'10.0.0.11/file.dat_3.data']},
'path': '10.0.0.10/file.dat_1.metadata'},
2:
{blocks: {4: ['10.0.0.9/file.dat_4.data',
'10.0.0.10/file.dat_4.data',
'10.0.0.11/file.dat_4.data'],
5: ['10.0.0.9/file.dat_5.data',
'10.0.0.10/file.dat_5.data',
'10.0.0.11/file.dat_5.data'],
6: ['10.0.0.9/file.dat_6.data',
'10.0.0.10/file.dat_6.data',
'10.0.0.11/file.dat_6.data']},
'path': '10.0.0.9/file.dat_2.metadata'}
}
After every block is mapped, the thread exemine if
the file can be restored. If the file can be restored,
the thread create missing parts and write the data to
the path in 'self.real_file'.
Raises:
Exception: Description
"""
self.logger.info(self.name + ' thread started')
# receiving all blocks until the clients finished sending.
start_time = time.time()
while not all(self.clients.values()) and time.time() < start_time + 30:
try:
message = self.restore_queue.get(timeout=3)
except:
self.logger.debug('waiting for blocks...')
else:
# if received a massage, reset timeout
start_time = time.time()
message_type = message['type']
self.logger.debug(
'received message of type \'%s\'' % message_type)
if message_type == 'block':
self.received_block(message)
elif message_type == 'file_sent':
self.clients[message['client']] = True
elif message_type == 'exit':
self.exit_thread(success=False)
return
else:
log = 'unknown message type: %s. message not processed'
self.logger.warning(log % message_type)
self.logger.info('finished collecting blocks of file %s' %
self.virtual_file)
# mapping the restored file blocks
mapping = {}
for path in self.get_blocks_names(block_type=protocol.METADATA_BLOCK):
basename = os.path.basename(path)
file_info = parse_file_name(basename)
number = file_info['number']
if number.isdigit():
number = int(number)
mapping[number] = {'path': path, 'blocks': {}}
# insert records for missing metadata blocks into the mapping
for metadata_number in xrange(1, self.validation_number + 1):
if metadata_number not in mapping:
mapping[metadata_number] = {'path': None, 'blocks': {}}
start_number = 1 + self.validation_level * (metadata_number - 1)
end_number = self.validation_level * metadata_number
# insert paths of data blocks matching he missing metadata block
for data_number in xrange(start_number, end_number + 1):
blocks = mapping[metadata_number]['blocks']
if data_number <= self.block_number:
blocks[data_number] = self.get_blocks_names(
block_type=protocol.DATA_BLOCK, number=data_number)
self.logger.debug('file blocks mapping:\n' + pprint.pformat(mapping))
# mapping missing / corrupted blocks
valid_blocks = {}
missing_data = {}
missing_metadata = []
validation_warning = True
for metadata_number in mapping:
metadata_path = mapping[metadata_number]['path']
blocks_dict = mapping[metadata_number]['blocks']
missing_data[metadata_number] = []
# check if the metadata can be used
valid_metadata = True
if metadata_path is not None:
try:
f = open(metadata_path, 'rb')
metadata = protocol.parse(f.read())
f.close()
if type(metadata) != dict:
raise Exception()
except:
valid_metadata = False
else:
valid_metadata = False
if valid_metadata:
# if the metadata can be used, validate the blocks against it
for block_number in blocks_dict:
if 'hashes' in metadata:
hashes = metadata['hashes']
if block_number in hashes:
block_hash = hashes[block_number]
block_list = blocks_dict[block_number]
if block_list == []:
missing_data[metadata_number].append(block_number)
for path in block_list:
f = open(path, 'rb')
content = f.read()
f.close()
validated_hash = encrypt.hash_string(content)
if validated_hash.lower() == block_hash.lower():
valid_blocks[block_number] = path
break
'''
if all variant of the blocks were tested against the metadata
and none of them are valid, then add the block to the missing
data. Later, try to recreate them
'''
for block_number in blocks_dict:
if block_number not in valid_blocks:
if block_number not in missing_data[metadata_number]:
missing_data[metadata_number].append(block_number)
else:
# if the metadata is not usable, choose the most common block
missing_metadata.append(metadata_number)
for block_number in blocks_dict:
if blocks_dict[block_number] != []:
content = []
for path in blocks_dict[block_number]:
f = open(path, 'rb')
content.append((path, f.read()))
f.close()
path, _ = max(set(content), key=content.count)
basename = os.path.basename(path)
number = parse_file_name(basename)['number']
number = int(number)
valid_blocks[number] = path
else:
missing_data[metadata_number].append(block_number)
if validation_warning:
self.logger.warning('some blocks are not validated')
validation_warning = False
self.logger.debug('valid blocks: %s' % pprint.pformat(valid_blocks))
self.logger.debug('missing data: %s' % pprint.pformat(missing_data))
self.logger.debug('missing metadata: %s' %
pprint.pformat(missing_metadata))
# check if file can be restored, and recreate missing parts
corrupted = False
for metadata_number in missing_data:
missing_count = len(missing_data[metadata_number])
# if more then one block is missing per metadata block,
# the file cannot be restored
if missing_count > 1:
corrupted = True
break
elif missing_count == 1:
# if only one block is missing per metadata block, rebuild it
if metadata_number not in missing_metadata:
# read metadata block
with open(mapping[metadata_number]['path'], 'rb') as f:
metadata = protocol.parse(f.read())
# calculate the first and last number of data block
# of the current metadata block
start_number = 1 + (self.validation_level *
(metadata_number - 1))
end_number = self.validation_level * metadata_number
if end_number > self.block_number:
end_number = self.block_number
# restore the missing block using the metadata
restored = metadata['xor']
missing_block_number = missing_data[metadata_number][0]
for block_number in xrange(start_number, end_number + 1):
if block_number != missing_block_number:
with open(valid_blocks[block_number], 'rb') as f:
content = f.read()
restored = encrypt.xor_strings(restored, content)
# if the hash of the block is valid, write it to file
restored_hash = encrypt.hash_string(restored)
metadata_hash = metadata['hashes'][missing_block_number]
if restored_hash.lower() == metadata_hash.lower():
file_name = build_file_name(
block_type=protocol.DATA_BLOCK,
name=self.virtual_file,
number=missing_block_number)
path = os.path.join(self.temp, file_name)
restored_block = open(path, 'wb')
restored_block.write(restored)
restored_block.close()
valid_blocks[missing_block_number] = path
else:
corrupted = True
break
else:
corrupted = True
break
# if file can be restored, restore it
if not corrupted:
real_file = open(self.real_file, 'wb')
for block_number in xrange(1, self.block_number + 1):
path = valid_blocks[block_number]
f = open(path, 'rb')
real_file.write(f.read())
f.close()
real_file.close()
self.logger.info('\'%s\' restored successfully' %
self.virtual_file)
# else, tell the logic thread
else:
err = '\'%s\' is corrupted, could not restore' % self.virtual_file
self.logger.error(err)
message = protocol.thread.error(thread_id=self.ident, message=err)
self.logic_queue.put(message)
# executing exit operations
self.callback()
self.exit_thread(success=not corrupted)
def testParseCmd(self):
testString = "er"
results = protocol.parse(testString)
self.assertEqual(results, ("er", []))
def testParseCmdManyArg(self):
testString = "er\trt\twere"
results = protocol.parse(testString)
self.assertEqual(results, ("er", ["rt", "were"]))
def testParseCmdArg(self):
testString = "er\trt"
results = protocol.parse(testString)
self.assertEqual(results, ("er", ["rt"]))
