class FileFormService(base_service.BaseService):
## FileFormService States
(
START_STATE,
HEADERS_STATE,
CONTENT_STATE,
FINAL_STATE,
) = range(4)
## Constructor for FileFormService
# @param entry (pollable) the entry (probably @ref
# common.pollables.service_socket) using the service
# @param pollables (dict) All the pollables currently in the server
# @param args (dict) Arguments for this service
def __init__(self, entry, pollables, args):
super(FileFormService, self).__init__(["Content-Type"])
## Current content recieved
self._content = ""
## Boundary which the form uses
self._boundary = None
## Fields of the form
self._fields = {}
## State Machine that service uses
self._state_machine = None
## File descriptor of file if file is part of the form
self._fd = None
## Filename required to upload in form
self._filename = None
## Temporary file name if something goes wrong
self._tmp_filename = constants.TMP_FILE_NAME
## Name of argument from form
self._arg_name = None
## Name of the service
# needed for Frontend purposes, creating clients
# required by common.services.base_service.BaseService
# @returns (str) service name
@staticmethod
def get_name():
return "/fileupload"
# STATE FUNCTIONS
## Start state after function. Checks if first boundary has been recieved
## @param entry (pollable) the entry that the service is assigned to
## @returns next_state (int) returns the next state of the state machine
def after_start(self, entry):
if self._content.find("--%s" % self._boundary) == -1:
return None
self._content = self._content.split(
"--%s%s" % (self._boundary, constants.CRLF_BIN), 1)[1]
return FileFormService.HEADERS_STATE
## Form headers state after function. Checks if got all headers then
## processes them
## @param entry (pollable) the entry that the service is assigned to
## @returns next_state (int) returns the next state of the state machine
def after_headers(self, entry):
lines = self._content.split(constants.CRLF_BIN)
if "" not in lines:
return None
# got all the headers, process them
headers = {}
for index in range(len(lines)):
line = lines[index]
if line == "":
self._content = constants.CRLF_BIN.join(lines[index + 1:])
break
k, v = util.parse_header(line)
headers[k] = v
if "Content-Disposition" not in headers.keys():
raise RuntimeError("Missing content-disposition header")
self._filename = None
self._arg_name = None
disposition_fields = headers["Content-Disposition"].replace(" ", "")
disposition_fields = disposition_fields.split(";")[1:]
for field in disposition_fields:
name, info = field.split('=', 1)
# the info part is surrounded by parenthesies
info = info[1:-1]
if name == "filename":
self._filename = info
self._fd = os.open(self._tmp_filename,
os.O_RDWR | os.O_CREAT | os.O_BINARY, 0o666)
elif name == "name":
self._arg_name = info
self._args[info] = [""]
return FileFormService.CONTENT_STATE
## Content state after function. Checks if got all content and calls
## arg_handle or file_handle according to current field
## @param entry (pollable) the entry that the service is assigned to
## @returns next_state (int) returns the next state of the state machine
def after_content(self, entry):
# first we must check if there are any more mid - boundaries
if self._content.find(post_util.mid_boundary(self._boundary)) != -1:
buf = self._content.split(
post_util.mid_boundary(self._boundary),
1,
)[0]
next_state = FileFormService.HEADERS_STATE
elif self._content.find(post_util.end_boundary(self._boundary)) != -1:
buf = self._content.split(
post_util.end_boundary(self._boundary),
1,
)[0]
next_state = FileFormService.FINAL_STATE
else:
buf = self._content
next_state = None
if self._filename is not None:
self.file_handle(buf, next_state)
else:
self.arg_handle(buf, next_state)
self._content = self._content[len(buf):]
if next_state == FileFormService.HEADERS_STATE:
self._content = self._content.split(
post_util.mid_boundary(self._boundary), 1)[1]
return next_state
## FileFormService State Machine States
STATES = [
state.State(
START_STATE,
[HEADERS_STATE],
after_func=after_start,
),
state.State(HEADERS_STATE, [CONTENT_STATE], after_func=after_headers),
state.State(CONTENT_STATE, [HEADERS_STATE, FINAL_STATE],
after_func=after_content),
state.State(FINAL_STATE, [FINAL_STATE])
]
## Before entry gets content service state. Check this is a multipart
## form-data. Initializes the StateMachine.
## @param entry (pollable) the entry that the service is assigned to
def before_content(self, entry):
content_type = entry.request_context["headers"]["Content-Type"]
if (content_type.find("multipart/form-data") == -1
or content_type.find("boundary") == -1):
raise RuntimeError("%s:\tBad Form Request%s" %
(entry, content_type))
self._boundary = content_type.split("boundary=")[1]
self._state_machine = state_machine.StateMachine(
FileFormService.STATES,
FileFormService.STATES[FileFormService.START_STATE],
FileFormService.STATES[FileFormService.FINAL_STATE])
## Handle content service function. Pass on to StateMachine functions
## @param entry (pollable) the entry that the service is assigned to
## @param content (string) current content that has been recieved
def handle_content(self, entry, content):
self._content += content
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Before entry sends the response_headers service state.
## @param entry (pollable) the entry that the service is assigned to
def before_response_headers(self, entry):
if self._response_status == 200:
self._response_content = html_util.create_html_page(
"File was uploaded successfully")
self._response_headers = {
"Content-Length": len(self._response_content),
}
return True
## Function that handles arguments from the FileFormService
## @param buf (string) buf read from socket
## @param next_state (int) if finished reading argument
def arg_handle(self, buf, next_state):
self._args[self._arg_name][0] += buf
## Function that handles files from the FileFormService
## @param buf (string) buf read from socket
## @param next_state (int) if finished reading file
def file_handle(self, buf, next_state):
while buf:
buf = buf[os.write(self._fd, buf):]
self._content = buf + self._content
if next_state:
os.rename(os.path.normpath(self._tmp_filename),
os.path.normpath(self._save_filename))
os.close(self._fd)
class DisconnectService(base_service.BaseService):
## Disconnect States
(
DISCONNECT_STATE,
FINAL_STATE,
) = range(2)
## Constructor for DisonnectService
# @param entry (pollable) the entry (probably @ref
# common.pollables.service_socket) using the service
# @param pollables (dict) All the pollables currently in the server
# @param args (dict) Arguments for this service
def __init__(self, entry, pollables, args):
super(DisconnectService, self).__init__([], ["disk_UUID"], args)
## Volume we're dealing with
self._volume = None
## Disks we're dealing with
self._disks = None
## Disk UUID of connected disk
self._disk_UUID = None
## Volume UUID of relevant volume
self._volume_UUID = None
## StateMachine object
self._state_machine = None
## pollables of the Frontend server
self._pollables = pollables
## Disk Manager that manages all the clients
self._disk_manager = None
## Name of the service
# needed for Frontend purposes, creating clients
# required by common.services.base_service.BaseService
# @returns (str) service name
@staticmethod
def get_name():
return "/disconnect"
## Before disconnecting the disk function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_disconnect(self, entry):
self._disk_UUID = self._args["disk_UUID"][0]
self._volume_UUID = self._args["volume_UUID"][0]
# extract the disks from the wanted volume
self._volume = entry.application_context["volumes"][self._volume_UUID]
self._disks = self._volume["disks"]
# check if disk is already disconnected
if self._disks[self._disk_UUID]["state"] != constants.ONLINE:
return True
# check that all other disks are online (RAID5 requirements)
for disk_UUID, disk in self._disks.items():
if not disk["state"] == constants.ONLINE:
raise RuntimeError(
"Can't turn disk %s offline, already have disk %s offline"
% (self._disk_UUID, disk_UUID))
# already set to offline so that another attempt to disconnect shall be
# denied
self._disks[self._disk_UUID]["state"] = constants.OFFLINE
self._disks[self._disk_UUID]["cache"] = cache.Cache(
mode=cache.Cache.CACHE_MODE)
# now need to increment other disks level
# check this isn't the disk we are disconnecting
self._disk_manager = disk_manager.DiskManager(
self._disks,
self._pollables,
entry,
service_util.create_update_level_contexts(
self._disks, {
disk_UUID: {
"addition": "1",
"password": self._volume["long_password"]
}
for disk_UUID in self._disks.keys()
if disk_UUID != self._disk_UUID
}),
)
return False # will always need input, not an epsilon_path
## After disconnecting the disk function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_disconnect(self, entry):
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError("Got bad status code from BDS")
# finished smoothly, update levels on frontend disks
for disk_UUID, disk in self._disks.items():
if disk_UUID != self._disk_UUID:
disk["level"] += 1
# also mark this disk as offline
self._disks[self._disk_UUID]["state"] = constants.OFFLINE
entry.state = constants.SEND_HEADERS_STATE
return DisconnectService.FINAL_STATE
## State Machine states
STATES = [
state.State(
DISCONNECT_STATE,
[FINAL_STATE],
before_disconnect,
after_disconnect,
),
state.State(FINAL_STATE, [FINAL_STATE])
]
## Before pollable sends response status service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_status(self, entry):
self._state_machine = state_machine.StateMachine(
DisconnectService.STATES,
DisconnectService.STATES[DisconnectService.DISCONNECT_STATE],
DisconnectService.STATES[DisconnectService.FINAL_STATE])
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Called when BDSClientSocket invoke the on_finsh method to wake up
## the ServiceSocket. Let StateMachine handle the wake up call.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
def on_finish(self, entry):
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Before pollable sends response headers service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_headers(self, entry):
# Re-send the management part. No refresh so user can enter new disk
self._response_content = html_util.create_html_page(
"",
constants.HTML_DISPLAY_HEADER,
0,
display_disks_service.DisplayDisksService.get_name(),
)
self._response_headers = {
"Content-Length": "%s" % len(self._response_content),
}
return True
class ReadFromDiskService(base_service.BaseService):
## Reading States
(READ_STATE, FINAL_STATE) = range(2)
## Reading Modes
(REGULAR, RECONSTRUCT) = range(2)
## Constructor for ReadFromDiskService
# @param entry (pollable) the entry (probably @ref
# common.pollables.service_socket) using the service
# @param pollables (dict) All the pollables currently in the server
# @param args (dict) Arguments for this service
def __init__(self, entry, pollables, args):
super(ReadFromDiskService, self).__init__(
["Content-Type", "Authorization"],
["volume_UUID", "disk_num", "firstblock", "blocks"], args)
## Reading mode
self._block_mode = ReadFromDiskService.REGULAR
## Current block_num we're reading
self._current_block = None
## physical UUID of the disk we're reading from
self._current_phy_UUID = None
## Logical disk num of disk we're reading from
self._disk_num = None
## UUID of volume we're dealing with
self._volume_UUID = None
## Volume we're dealing with
self._volume = None
## Disks we're dealing with
self._disks = None
## StateMachine object
self._state_machine = None
## pollables of the Frontend server
self._pollables = pollables
## Disk Manager that manages all the clients
self._disk_manager = None
## Name of the service
# needed for Frontend purposes, creating clients
# required by common.services.base_service.BaseService
# @returns (str) service name
@staticmethod
def get_name():
return "/disk_read"
## Before reading a block from the Block Devices.
## First try reading the block regularly. If got DiskRefused, Then read
## that block from all the other disks and compute the missing block.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_read(self, entry):
self._current_phy_UUID = disk_util.get_physical_disk_UUID(
self._disks, self._disk_num, self._current_block)
try:
# First check availablity
available_disks = entry.application_context["available_disks"]
online, offline = util.sort_disks(available_disks)
if self._current_phy_UUID not in online.keys():
raise util.DiskRefused(self._current_phy_UUID)
self._block_mode = ReadFromDiskService.REGULAR
self._disk_manager = disk_manager.DiskManager(
self._disks,
self._pollables,
entry,
service_util.create_get_block_contexts(
self._disks, {
self._current_phy_UUID: {
"block_num": self._current_block,
"password": self._volume["long_password"]
}
}),
)
except util.DiskRefused as e:
# probably got an error when trying to reach a certain BDS
# ServiceSocket. We shall try to get the data from the rest of
# the disks. Otherwise, two disks are down and theres nothing
# we can do
logging.debug(
"%s:\t Couldn't connect to one of the BDSServers, %s: %s" %
(entry, self._current_phy_UUID, e))
try:
self._block_mode = ReadFromDiskService.RECONSTRUCT
# create request info for all the other disks
request_info = {}
for disk_UUID in self._disks.keys():
if disk_UUID != self._current_phy_UUID:
request_info[disk_UUID] = {
"block_num": self._current_block,
"password": self._volume["long_password"]
}
self._disk_manager = disk_manager.DiskManager(
self._disks,
self._pollables,
entry,
service_util.create_get_block_contexts(
self._disks, request_info),
)
except socket.error as e:
# Got another bad connection (Connection refused most likely)
raise RuntimeError(("%s:\t Couldn't connect to two of the" +
"BDSServers, giving up: %s") % (entry, e))
entry.state = constants.SLEEPING_STATE
return False # always need input, not an epsilon path
## After reading from relevant block devices.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_read(self, entry):
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError("Got bad status code from BDS")
# if we have a pending block, send it back to client
# Get ready for next block (if there is)
# TODO: Too much in response_content
self.update_block()
self._current_block += 1
entry.state = constants.SEND_CONTENT_STATE
if (self._current_block == (int(self._args["firstblock"][0]) +
int(self._args["blocks"][0]))):
return ReadFromDiskService.FINAL_STATE
return ReadFromDiskService.READ_STATE
## Function that updates the response content with the computed block.
def update_block(self):
client_responses = self._disk_manager.get_responses()
# regular block update
if self._block_mode == ReadFromDiskService.REGULAR:
self._response_content += (
client_responses[self._current_phy_UUID]["content"].ljust(
constants.BLOCK_SIZE, chr(0)))
# reconstruct block update
elif self._block_mode == ReadFromDiskService.RECONSTRUCT:
blocks = []
for disk_num, response in client_responses.items():
blocks.append(response["content"])
self._response_content += disk_util.compute_missing_block(
blocks).ljust(constants.BLOCK_SIZE, chr(0))
## Reading states for StateMachine
STATES = [
state.State(READ_STATE, [READ_STATE, FINAL_STATE], before_read,
after_read),
state.State(FINAL_STATE, [FINAL_STATE])
]
# Before pollable sends response status service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_status(self, entry):
# first check login
if not util.check_user_login(entry):
# login was unsucsessful, notify the user agent
self._response_status = 401
self._response_headers[
"WWW-Authenticate"] = "Basic realm='myRealm'"
return True
# login went smoothly, moving on to disk read
self._disk_num = int(self._args["disk_num"][0])
self._volume_UUID = self._args["volume_UUID"][0]
# first check validity of volume_UUID
if (self._volume_UUID
not in entry.application_context["volumes"].keys()
or (entry.application_context["volumes"][self._volume_UUID]
["volume_state"] != constants.INITIALIZED)):
raise RuntimeError("%s:\t Need to initialize volume" % (entry, ))
self._volume = entry.application_context["volumes"][self._volume_UUID]
self._disks = self._volume["disks"]
# now check validity of disk_num
if self._disk_num < 0 or self._disk_num >= len(self._disks) - 1:
raise RuntimeError("%s:\t Logical disk not part of volume %s " % (
entry,
self._disk_num,
))
# also checek validity of blocks requested
if int(self._args["blocks"][0]) < 0:
raise RuntimeError("%s:\t Invalid amount of blocks: %s" %
(entry, self._args["blocks"][0]))
elif int(self._args["firstblock"][0]) < 0:
raise RuntimeError("%s:\t Invalid first block requested: %s" %
(entry, self._args["firstblock"][0]))
# could check on how many are active...
self._response_headers = {
"Content-Length":
(int(self._args["blocks"][0]) * constants.BLOCK_SIZE),
"Content-Type":
"text/html",
"Content-Disposition":
("attachment; filename=blocks[%s : %s].txt" %
(int(self._args["firstblock"][0]),
(int(self._args["blocks"][0]) +
int(self._args["firstblock"][0])))),
}
self._current_block = int(self._args["firstblock"][0])
# initialize state machine for reading
first_state = ReadFromDiskService.READ_STATE
if int(self._args["blocks"][0]) == 0:
first_state = ReadFromDiskService.FINAL_STATE
self._state_machine = state_machine.StateMachine(
ReadFromDiskService.STATES,
ReadFromDiskService.STATES[first_state],
ReadFromDiskService.STATES[ReadFromDiskService.FINAL_STATE])
return True
## Before pollable sends response content service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_content(self, entry):
# first check if we have an error and we don't want to read
if self._response_status != 200:
return True
# pass args to the machine, will use *args to pass them on
# if the machine returns True, we know we can move on
return self._state_machine.run_machine((self, entry))
## Called when BDSClientSocket invoke the on_finish method to wake up
## the ServiceSocket. Let StateMachine handle the wake up call.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
def on_finish(self, entry):
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
class InitService(base_service.BaseService):
## Initialization modes
(
EXISTING_MODE,
SCRATCH_MODE,
) = range(2)
## Initialization satates
(
SETUP_STATE,
LOGIN_STATE,
EXISTING_MOUNT_STATE,
SCRATCH_MOUNT_STATE,
FINAL_STATE,
) = range(5)
## Constructor for InitService
# @param entry (pollable) the entry (probably @ref
# common.pollables.service_socket) using the service
# @param pollables (dict) All the pollables currently in the server
# @param args (dict) Arguments for this service
def __init__(self, entry, pollables, args):
super(InitService, self).__init__()
## Http arguments
self._args = args # args will be checked independently
## Pollables of the Frontend server
self._pollables = pollables
## Volume we're dealing with
self._volume = None # still not sure what volume we're dealing with
## Initialization mode
self._mode = None
## StateMachine object
self._state_machine = None
## Disk Manager that manages all the clients
self._disk_manager = None
## Name of the service
# needed for Frontend purposes, creating clients
# required by common.services.base_service.BaseService
# @returns (str) service name
@staticmethod
def get_name():
return "/init"
# STATE FUNCTIONS
## Before setup function. check what volume is required.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_setup(self, entry):
# create a list of available_disks
new_disks = []
for arg_name, arg_info in self._args.items():
new_disks.append(arg_info[0])
# first check the number of disks requested to create a volume
if len(new_disks) < 2:
raise RuntimeError(
"%s:\tNot enough disks for a RAID5 volume (need at least 2): %s"
% (entry, len(new_disks)))
# check if all the disks are available_disks
for disk_UUID in new_disks:
if disk_UUID not in entry.application_context[
"available_disks"].keys():
raise RuntimeError(
"%s:\t One or more of the disks is offline" % entry)
# check all the disks have the same volume_UUID,
# either an empty string or something else.
common_UUID = entry.application_context["available_disks"][
new_disks[0]]["volume_UUID"]
for disk_UUID in new_disks:
if (entry.application_context["available_disks"][disk_UUID]
["volume_UUID"] != common_UUID):
raise RuntimeError("%s:\t Got two different volumes" % entry)
# check if volume_UUID is in the system (if not "")
if common_UUID == "":
self._mode == InitService.SCRATCH_MODE
# create a new volume_UUID and write in config_file
volume_UUID = util.generate_uuid()
long_password = util.generate_password()
# add volume to list of all volumes
entry.application_context["volumes"][volume_UUID] = {
"volume_UUID": volume_UUID,
"volume_state": constants.INITIALIZED,
"long_password": long_password,
"disks": {},
}
# update the config_file with the new volume
config_util.write_section_config(
entry.application_context["config_file"],
"volume%s" % len(entry.application_context["volumes"]), {
"volume_UUID": volume_UUID,
"long_password": long_password
})
else:
self._mode = InitService.EXISTING_MODE
# use existing volume uuid and long password
volume_UUID = entry.application_context["available_disks"][
new_disks[0]]["volume_UUID"]
long_password = entry.application_context["volumes"][volume_UUID][
"long_password"]
# check if UUID is in config file sections
if volume_UUID not in entry.application_context["volumes"].keys():
raise RuntimeError("%s:\tUnrecognized existing volume")
# update volume state
entry.application_context["volumes"][volume_UUID][
"volume_state"] = constants.INITIALIZED
# update the initialized volume index
entry.application_context["volumes"][volume_UUID]["volume_num"] = (len(
util.initialized_volumes(entry.application_context["volumes"])))
# add the new disks to this volume:
for disk_num in range(len(new_disks)):
# retrieve the disk_UUID
disk_UUID = new_disks[disk_num]
# create a declared_disk from the available_disks
declared_disk = entry.application_context["available_disks"][
disk_UUID]
# add the new disk to the volume. key is disk UUID (arg_content)
entry.application_context["volumes"][volume_UUID]["disks"][
disk_UUID] = {
"disk_UUID": disk_UUID,
"disk_num": disk_num,
"address": declared_disk["TCP_address"],
"volume_UUID": volume_UUID,
"state": constants.STARTUP,
"cache": cache.Cache(),
"level": "",
"peers": "",
}
# finally we have our disks. Update as an attribute
self._volume = entry.application_context["volumes"][volume_UUID]
# this is an epsilon path, just setting up
return True
# Before login to the block devices in the new volume (set password)
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_login(self, entry):
if self._mode == InitService.EXISTING_MODE:
# no need to login, already updated
return True # epsilon_path
# need to login to new block device
self._disk_manager = disk_manager.DiskManager(
self._volume["disks"], self._pollables, entry,
service_util.create_login_contexts(self._volume, ))
entry.state = constants.SLEEPING_STATE
return False # will always need input, not an epsilon_path
## After login to each of the Block Devices
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_login(self, entry):
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError(
"Block Device Server sent a bad status code: %s" %
(self._disk_manager.get_responses()[0]["status"]))
else:
# must be in SCRATCH_MODE. return next state
return InitService.SCRATCH_MOUNT_STATE
## Before we mount an existing volume
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_existing_mount(self, entry):
self._disk_manager = disk_manager.DiskManager(
self._volume["disks"], self._pollables, entry,
service_util.create_get_disk_info_contexts(
self._volume["disks"],
self._volume["disks"].keys(),
))
entry.state = constants.SLEEPING_STATE
return False # will always need input, not an epsilon_path
## After we mount an existing volume
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_existing_mount(self, entry):
# check if got a response from ALL of the
# block devices:
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError(
"%s:\tBlock Device Server sent a bad status code")
# check the actual response
client_responses = self._disk_manager.get_responses()
# make the disks_data easily accessible
disks_data = []
for disk_UUID in self._volume["disks"].keys():
disks_data.append(client_responses[disk_UUID]["content"].split(
constants.MY_SEPERATOR))
# check if a disk need to be rebuilt
rebuild_disk = None
for disk_num in range(len(self._volume["disks"])):
# first lets check the volume_UUID:
if disks_data[disk_num][1] != disks_data[0][1]:
raise RuntimeError("Got two different topoligies: \n%s\n%s" %
(disks_data[disk_num][1], disks_data[0][1]))
# next lets check the generation level
# By RAID5, we can only rebuild one disk at a time
if disks_data[disk_num][0] != disks_data[0][0]:
raise RuntimeError(
"Initialize only a consistent set (level mixup): %s != %s"
% (disks_data[disk_num][0], disks_data[0][0]))
# Lets now check the disk UUID:
if disks_data[disk_num][3:].count(disks_data[disk_num][2]) != 1:
raise RuntimeError("Disk UUID shows up an invalid amount" +
" of times in peers %s" % (disk_num))
# And finally, check all the peers UUID's:
if disks_data[disk_num][3:] != disks_data[0][3:]:
raise RuntimeError("Unsynced peers")
# All the checks came back positive, ready to update disks
for disk_UUID, disk in self._volume["disks"].items():
self._volume["disks"][disk_UUID]["level"] = int(
disks_data[disk_num][0])
self._volume["disks"][disk_UUID]["peers"] = disks_data[disk_num][
3:]
self._volume["disks"][disk_UUID]["state"] = constants.ONLINE
entry.state = constants.SEND_CONTENT_STATE
return InitService.FINAL_STATE
## Before we mount a new volume from scratch
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_scratch_mount(self, entry):
peers = self._volume["disks"].keys() # for order create var
request_info = {}
for disk_UUID in self._volume["disks"].keys():
boundary = post_util.generate_boundary()
request_info[disk_UUID] = {
"boundary":
boundary,
"content":
self.create_disk_info_content(
boundary,
disk_UUID,
peers,
)
}
# update final disk stats
for disk_UUID, disk in self._volume["disks"].items():
self._volume["disks"][disk_UUID]["level"] = 0
self._volume["disks"][disk_UUID]["peers"] = peers
self._volume["disks"][disk_UUID]["state"] = constants.ONLINE
# create a disk manager
self._disk_manager = disk_manager.DiskManager(
self._volume["disks"], self._pollables, entry,
service_util.create_set_disk_info_contexts(self._volume["disks"],
request_info))
entry.state = constants.SLEEPING_STATE
return False # need input, not an epsilon_path
## After we mount a new volume from scratch
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_scratch_mount(self, entry):
# check if got a response from ALL of the
# block devices:
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
# Should delete the disk array, initilization has failed
raise RuntimeError("Initilization failed" +
"Block Device Server sent a bad status code")
entry.state = constants.SEND_CONTENT_STATE
return InitService.FINAL_STATE
## Initilization states for StateMachine
STATES = [
state.State(SETUP_STATE, [LOGIN_STATE], before_func=before_setup),
state.State(
LOGIN_STATE,
[EXISTING_MOUNT_STATE, SCRATCH_MOUNT_STATE], # order matters
before_login,
after_login,
),
state.State(EXISTING_MOUNT_STATE, [FINAL_STATE], before_existing_mount,
after_existing_mount),
state.State(SCRATCH_MOUNT_STATE, [FINAL_STATE], before_scratch_mount,
after_scratch_mount),
state.State(FINAL_STATE, [FINAL_STATE])
]
## Before pollable sends response status service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_status(self, entry):
# create initilization state machine
self._state_machine = state_machine.StateMachine(
InitService.STATES, InitService.STATES[InitService.SETUP_STATE],
InitService.STATES[InitService.FINAL_STATE])
return True
## Before pollable sends response headers service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_headers(self, entry):
# pass args to the machine, will use *args to pass them on
return self._state_machine.run_machine((self, entry))
## Before pollable sends response content service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_content(self, entry):
# Re-send the management part
self._response_content = html_util.create_html_page(
"",
constants.HTML_DISPLAY_HEADER,
0,
display_disks_service.DisplayDisksService.get_name(),
)
self._response_headers = {
"Content-Length": "%s" % len(self._response_content),
}
return True
## Called when BDSClientSocket invoke the on_finsh method to wake up
## the ServiceSocket. Let StateMachine handle the wake up call.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
def on_finish(self, entry):
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Create the dictionary that can be made into POST method content
## multipart form-data
## @param boundary (string) boundary we're using
## @param disk_uuid (string) current UUID of disk we're initializing
## @param disks_uuids (list) list of all the uuids of the disks peers
def create_disk_info_content(
self,
boundary,
disk_uuid,
disks_uuids,
):
# actual file content
# disk info files are of the following format:
# level \r\n
# volume_UUID \r\n
# disk_uuid \r\n
# peer_uuids \r\n
return post_util.make_post_content(
boundary, {
("Content-Disposition : form-data; filename='irrelevant'"):
(("%s" * 7) %
(0, constants.MY_SEPERATOR, self._volume["volume_UUID"],
constants.MY_SEPERATOR, disk_uuid, constants.MY_SEPERATOR,
constants.MY_SEPERATOR.join(disks_uuids)))
})
class ConnectService(base_service.BaseService):
## Constructor for ConnectService
# @param entry (pollable) the entry (probably @ref
# common.pollables.service_socket) using the service
# @param pollables (dict) All the pollables currently in the server
# @param args (dict) Arguments for this service
def __init__(self, entry, pollables, args):
super(ConnectService, self).__init__([], ["disk_UUID", "volume_UUID"],
args)
## Volume we're dealing with
self._volume = None
## Disks we're dealing with
self._disks = None
## Disk UUID of connected disk
self._disk_UUID = None
## Mode of adding a new disk
self._new_disk_mode = False
## Disk already built boolean
self._disk_built = False
## StateMachine object
self._state_machine = None
## Current block num (for rebuilding)
self._current_block_num = ""
## Current data (for rebuilding)
self._current_data = ""
## pollables of the Frontend server
self._pollables = pollables
## Disk Manager that manages all the clients
self._disk_manager = None
## Name of the service
# needed for Frontend purposes, creating clients
# required by common.services.base_service.BaseService
# @returns (str) service name
@staticmethod
def get_name():
return "/connect"
## Checks if and how the disk needs to be rebuilt.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns need_rebuild (bool) if needs to be rebuilt
def initial_setup(self, entry):
self._disk_UUID = self._args["disk_UUID"][0]
self._volume_UUID = self._args["volume_UUID"][0]
# first check validity of volume
if (self._volume_UUID
not in entry.application_context["volumes"].keys()
or (entry.application_context["volumes"][self._volume_UUID]
["volume_state"] != constants.INITIALIZED)):
raise RuntimeError("%s:\t Need to initialize volume" % (entry, ))
self._volume = entry.application_context["volumes"][self._volume_UUID]
self._disks = self._volume["disks"]
# now check validity of disk_UUID
if self._disk_UUID not in self._disks.keys():
raise RuntimeError("%s:\t Disk not part of volume" % (entry, ))
# sanity check that this level is no more than all the others:
for disk_UUID, disk in self._disks.items():
if (disk_UUID != self._disk_UUID and
(disk["level"] < self._disks[self._disk_UUID]["level"])):
raise RuntimeError("Error in levels")
self._disks[self._disk_UUID]["state"] = constants.REBUILD
## Before pollable sends response status service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_response_status(self, entry):
# initial_setup, also check if we need to add thid disk out of no-where
self.initial_setup(entry)
# Re-send the management part
self._response_content = html_util.create_html_page(
"",
constants.HTML_DISPLAY_HEADER,
0,
display_disks_service.DisplayDisksService.get_name(),
)
self._response_headers = {
"Content-Length": "%s" % len(self._response_content),
}
return True
# REBULD PART, DONE BEFORE TERMINATE (AFTER CLOSE)
## Rebuilding States
(GET_DATA_STATE, SET_DATA_STATE, UPDATE_LEVEL_STATE,
FINAL_STATE) = range(4)
# STATE FUNCTIONS:
## Before we get the rebulding data
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_get_data(self, entry):
self._current_block_num, self._current_data = (
self._disks[self._disk_UUID]["cache"].next_block())
if self._current_data is not None:
# got data stored in cache, no need for hard rebuild
# ==> This is an epsilon_path
return True
else:
# need to retreive data from XOR of all the disks besides the current
# in order to rebuild it
request_info = {}
for disk_UUID in self._disks.keys():
if disk_UUID != self._disk_UUID:
request_info[disk_UUID] = {
"block_num": self._current_block_num,
"password": self._volume["long_password"]
}
self._disk_manager = disk_manager.DiskManager(
self._disks, self._pollables, entry,
service_util.create_get_block_contexts(self._disks,
request_info))
entry.state = constants.SLEEPING_STATE
return False # need input, not an epsilon path
## After we get the rebulding data
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_get_data(self, entry):
# first check if the data has come from the cache
if self._current_data is not None:
return ConnectService.SET_DATA_STATE
# now we know that the data has come from the other disks. check if
# they all finished and their responses
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError("Block Device Server sent a bad status code")
# data not saved in cache, need to xor all the blocks
blocks = []
for disk_num, response in self._disk_manager.get_responses().items():
blocks.append(response["content"])
# check if finished scratch mode for cache
if ((self._disks[self._disk_UUID]["cache"].mode
== cache.Cache.SCRATCH_MODE) and disk_util.all_empty(blocks)):
# all the blocks we got are empty, change to cache mode
self._disks[self._disk_UUID]["cache"].mode = (
cache.Cache.CACHE_MODE)
# nothing to set now, we stay in GET_DATA_STATE and start working
# from cache
return ConnectService.GET_DATA_STATE
else:
self._current_data = disk_util.compute_missing_block(blocks)
return ConnectService.SET_DATA_STATE
## Before we set the rebulding data
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_set_data(self, entry):
self._disk_manager = disk_manager.DiskManager(
self._disks, self._pollables, entry,
service_util.create_set_block_contexts(
self._disks, {
self._disk_UUID: {
"block_num": self._current_block_num,
"content": self._current_data,
"password": self._volume["long_password"]
}
}))
entry.state = constants.SLEEPING_STATE
return False # need input, not an epsilon path
## After we set the rebulding data
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_set_data(self, entry):
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError("Block Device Server sent a bad status code")
if self.check_if_built():
return ConnectService.UPDATE_LEVEL_STATE
return ConnectService.GET_DATA_STATE
## Before we update the level of the updated disk
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns epsilon_path (bool) if there is no need for input
def before_update_level(self, entry):
self._disk_manager = disk_manager.DiskManager(
self._disks, self._pollables, entry,
service_util.create_update_level_contexts(
self._disks, {
self._disk_UUID: {
"addition": "1",
"password": self._volume["long_password"]
}
}))
entry.state = constants.SLEEPING_STATE
return False # need input, not an epsilon path
## Before we have updated the level of the updated disk
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns next_state (int) next state of StateMachine. None if not
## ready to move on to next state.
def after_update_level(self, entry):
if not self._disk_manager.check_if_finished():
return None
if not self._disk_manager.check_common_status_code("200"):
raise RuntimeError("Block Device Server sent a bad status code")
self._disks[self._disk_UUID]["level"] += 1
self._disks[self._disk_UUID]["state"] = constants.ONLINE
entry.state = constants.CLOSING_STATE
return ConnectService.FINAL_STATE
## Rebuilding states for StateMachine
STATES = [
state.State(
GET_DATA_STATE,
[SET_DATA_STATE],
before_get_data,
after_get_data,
),
state.State(
SET_DATA_STATE,
[GET_DATA_STATE, UPDATE_LEVEL_STATE],
before_set_data,
after_set_data,
),
state.State(
UPDATE_LEVEL_STATE,
[FINAL_STATE],
before_update_level,
after_update_level,
),
state.State(
FINAL_STATE,
[FINAL_STATE],
),
]
## Before pollable terminates service function
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
## @returns finished (bool) returns true if finished
def before_terminate(self, entry):
# create the state machine for rebuilding disk
first_state_index = ConnectService.GET_DATA_STATE
if self._new_disk_mode:
first_state_index = ConnectService.NEW_DISK_SETUP_STATE
elif self.check_if_built():
first_state_index = ConnectService.UPDATE_LEVEL_STATE
# create rebuild state machine
self._state_machine = state_machine.StateMachine(
ConnectService.STATES, ConnectService.STATES[first_state_index],
ConnectService.STATES[ConnectService.FINAL_STATE])
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Called when BDSClientSocket invoke the on_finsh method to wake up
## the ServiceSocket. Let StateMachine handle the wake up call.
## @param entry (@ref common.pollables.pollable.Pollable) entry we belong
## to
def on_finish(self, entry):
# pass args to the machine, will use *args to pass them on
self._state_machine.run_machine((self, entry))
## Checks if self._disk_UUID is built
## @returns built (bool) if disk needs to be rebuilt
def check_if_built(self):
# check if already connected, no need to rebuild, or cache is empty
if (self._disks[self._disk_UUID]["state"] == constants.REBUILD
and not self._disks[self._disk_UUID]["cache"].is_empty()):
return False
return True