async def rd_len(self, reader):
"""
Read 8 bytes for length information.
"""
bl.debug("Called 'rd_len(reader)'")
return await reader.read(8)
def _periodic_index_update_executor(self):
"""
Update the index in periodic intervals.
Executor thread.
"""
global LI_LOCK
global LOCAL_INDEX
while True:
get_index_event = self._comm_dict["get_index_event"]
receive_index_data_queue = self._comm_dict["get_index_data_queue"]
get_index_event.set()
with LI_LOCK:
try:
bl.debug("Waiting for index")
index = receive_index_data_queue.get(True, 100)
except queue.Empty as e:
bl.warning("Took more than 100 seconds to wait for "
"index ({}). There will be nothing on display "
"here.".format(e))
else:
LOCAL_INDEX = index["index"]
# if self._shutdown_event.wait(120): # update every 2 minutes
if self._shutdown_event.wait(): # wait forever, do not periodically update the index
return
def _ceph_data_executor(self):
"""
Run this in a separate executor.
"""
while True:
if self._shutdown_backend_manager_event.is_set():
return
try:
ans = self._file_content_name_hash_server_queue.get(True, .1)
except queue.Empty:
pass
else:
request_dict = ans
obj_key = request_dict["object"]
obj_namespace = request_dict["namespace"]
object_descriptor = "{}/{}".format(obj_namespace, obj_key)
bl.debug("Reading {} and making available".format(
object_descriptor))
occurence_key = object_descriptor
occurence_dict = {
"timestamp": time.time(),
"request_dict": request_dict
}
with self._ceph_data_lock:
self._ceph_data_dict[occurence_key] = occurence_dict
def _periodic_ceph_file_deletion_executor(self):
"""
Periodically delete old data in the ceph data dictionary.
The executor.
"""
while True:
# wait for 1 second, this is essentially a 1 second interval timer
if self._shutdown_backend_manager_event.wait(1):
return
current_time = time.time()
with self._ceph_data_lock:
for object_descriptor in list(self._ceph_data_dict.keys()):
timestamp = self._ceph_data_dict[object_descriptor][
"timestamp"]
elapsed_time = current_time - timestamp
if (elapsed_time > 60):
bl.debug("Removing {} after 60 seconds".format(
object_descriptor))
del self._ceph_data_dict[object_descriptor]
def __init__(self, event_loop, comm_dict):
bl.debug("Starting ProxyIndex")
self._loop = event_loop
self._comm_dict = comm_dict
self._shutdown_event = comm_dict["shutdown_platt_gateway_event"]
def dataset(self, dataset_hash):
"""
Return a dataset object for working with.
Args:
dataset_hash: The unique identifier for the dataset we want to access.
Returns:
_DatasetPrototype: The dataset that we want to access.
Raises:
TypeError: If ``type(dataset_hash)`` is not `str`.
"""
if not isinstance(dataset_hash, str):
raise TypeError('dataset_hash is {}, expected str'.format(
type(dataset_hash).__name__))
try:
# Return the object
return self._dataset_list[dataset_hash]
except KeyError as e:
bl.debug_warning(
'dataset_hash does not fit any dataset in scene: {}'.format(e))
raise ValueError('dataset_hash does not fit any dataset in scene')
async def _new_file_information_coro(self, reader, writer):
"""
Coroutine for sending information about new files to the client.
Checks the queue for new files in a separate executor, when this
executor returns a dictionary it gets sent to the client on the
registered connection.
"""
self._cancel_new_file_executor_event = threading.Event()
new_file_watchdog = self._loop.create_task(
self._check_file_connection(reader, writer))
# while the connection is open ...
while not reader.at_eof():
# check the queue in a separate executor
new_file_in_queue = await self._loop.run_in_executor(
None, self._check_file_queue)
if not new_file_in_queue:
return
bl.debug("Received info for {} for sending via socket".format(
new_file_in_queue))
await self._inform_client_new_file(reader, writer,
new_file_in_queue)
def delete_dataset(self, dataset_hash):
"""
Remove one dataset from the scene.
Args:
dataset_hash: The unique identifier for the dataset we want to delete.
Returns:
list: The remaining datasets in the scene.
Raises:
TypeError: If ``type(dataset_hash)`` is not `str`.
ValueError: If `dataset_hash` does not fit any dataset in the scene.
"""
if not isinstance(dataset_hash, str):
raise TypeError('dataset_hash is {}, expected str'.format(
type(dataset_hash).__name__))
try:
self._dataset_list.pop(dataset_hash)
# Delegate returning of the remainder to the standard method
return self.list_datasets()
except KeyError as e:
bl.debug_warning(
'dataset_hash does not fit any dataset in scene: {}'.format(e))
raise ValueError('dataset_hash does not fit any dataset in scene')
def _watch_incoming_files_executor(self):
"""
Enter new files into the global list.
"""
while True:
if self._shutdown_event.is_set():
return
try:
ans = self._file_request_answer_queue.get(True, .1)
except queue.Empty:
pass
else:
request_dict = ans["file_request"]
obj_key = request_dict["object"]
obj_namespace = request_dict["namespace"]
object_descriptor = "{}/{}".format(obj_namespace, obj_key)
bl.debug("Reading {} and making available".format(
object_descriptor))
occurence_key = object_descriptor
occurence_dict = {
"timestamp": time.time(),
"request_dict": request_dict
}
with GW_LOCK:
GATEWAY_DATA[occurence_key] = occurence_dict
def _periodic_file_deletion_executor(self):
"""
Periodically delete old data in the GATEWAY_DATA dictionary.
The executor.
"""
while True:
# wait for 1 second, this is essentially a 1 second interval timer
if self._shutdown_event.wait(1):
return
current_time = time.time()
with GW_LOCK:
for object_descriptor in list(GATEWAY_DATA.keys()):
timestamp = GATEWAY_DATA[object_descriptor]["timestamp"]
elapsed_time = current_time - timestamp
if (elapsed_time > 60):
bl.debug("Removing {} after 60 seconds".format(
object_descriptor))
del GATEWAY_DATA[object_descriptor]
async def _index_request_coro(self, reader, writer):
"""
Listens for index requests and answers them.
Listens for incoming data. If request for index is spotted we obtain the
index from the local data copy and return this to the client.
"""
# while the connection is open ...
while not reader.at_eof():
# wait for incoming data from the client
res = await self.read_data(reader, writer)
if not res:
await self.send_nack(writer)
return
await self.send_ack(writer)
if res["todo"] == "index":
bl.debug("Received index request")
# tell the local data copy that we request the index (index
# event)
self._get_index_server_event.set()
index = self._index_data_queue.get(True,
10) # wait up to 10 seconds
await self._send_index_to_client(reader, writer, index)
async def _send_file_to_client(self, reader, writer, file_dictionary):
"""
Answer file requests.
Encode the binary data in the file_dictionary as a base64 string. This
has to be reversed on the other side.
"""
connection_info = writer.get_extra_info('peername')
p_host = connection_info[0]
p_port = connection_info[1]
out_dict = dict()
out_dict["namespace"] = file_dictionary["namespace"]
out_dict["object"] = file_dictionary["object"]
out_dict["contents"] = base64.b64encode(
file_dictionary["value"]).decode()
out_dict["tags"] = file_dictionary["tags"]
bl.debug("Sending {}/{} to client [{}]".format(out_dict["namespace"],
out_dict["object"],
p_port))
todo_val = "file_request"
request_answer_dictionary = {"todo": todo_val, todo_val: out_dict}
await self._send_dictionary(reader, writer, request_answer_dictionary)
async def rd_data(self, reader, length):
"""
Read exactly the specified amount of bytes.
"""
bl.debug("Called 'rd_data(reader, length)'")
return await reader.readexactly(length)
def setup_logging(logging_level):
"""
Setup the loggers.
"""
gl(logging_level) # setup simulation logging
gl.info("Started Gateway logging with level '{}'".format(logging_level))
bl(logging_level) # setup backend logging
bl.info("Started Backend logging with level '{}'".format(logging_level))
def __init__(self, comm_dict):
bl.debug("Starting ProxyServices")
self._comm_dict = comm_dict
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._pi = pi.ProxyIndex(self._loop, self._comm_dict)
self._pd = pd.ProxyData(self._loop, self._comm_dict)
watch_incoming_files_task = self._loop.create_task(
self._pd._watch_incoming_files_coro())
periodically_delete_files_task = self._loop.create_task(
self._pd._periodic_file_deletion_coro())
periodically_update_index_task = self._loop.create_task(
self._pi._periodic_index_update_coro())
watch_new_files_task = self._loop.create_task(
self._pi._watch_new_files_coro())
self._shutdown_event = self._comm_dict["shutdown_platt_gateway_event"]
subscription_crawler_task = self._loop.create_task(
pi._subscription_crawler_coro(self._shutdown_event))
self._tasks = [
watch_incoming_files_task,
periodically_delete_files_task,
periodically_update_index_task,
watch_new_files_task,
subscription_crawler_task
]
try:
# start the tasks
self._loop.run_until_complete(asyncio.wait(self._tasks))
except KeyboardInterrupt:
pass
finally:
self._loop.stop()
all_tasks = asyncio.Task.all_tasks()
for task in all_tasks:
task.cancel()
with suppress(asyncio.CancelledError):
self._loop.run_until_complete(task)
self._loop.close()
bl.debug("ProxyServices is shut down")
def new_scene(self, dataset_list):
"""
Create a new scene with an object.
This adds a ScenePrototype to `self._scene_list`.
Args:
dataset_list (list (of str)): The path to the datasets we want to
instantiate a new scene with.
Returns:
None, dict: `None` if no dataset could be added to a new scene and a
dict with information what could be added and what not in the case
that we could add dataset(s) to a new scene.
Raises:
TypeError: If ``type(object_path)`` is not `list`.
Todo:
Make it impossible to create an empty scene.
"""
# Type checking for dataset_list
if not isinstance(dataset_list, list):
raise TypeError('dataset_list is {}, expected list'.format(
type(dataset_list).__name__))
# Do nothing if the dataset list is empty
if len(dataset_list) == 0:
return None
# See which datasets are valid
valid_datasets = []
available_datasets = (
self.list_available_datasets()['availableDatasets'])
for dataset in dataset_list:
if dataset in available_datasets:
valid_datasets.append(dataset)
# If there are no valid datasets to be added return None
if len(valid_datasets) == 0:
return None
try:
# Get a new instance of a scene
new_scene = _ScenePrototype(source_dict=self.source)
new_scene_hash = new_scene.name()
self._scene_list[new_scene_hash] = new_scene
# Here still dataset_list, so we can have a addDatasetFail entry
return_dict = self.add_datasets(new_scene_hash, dataset_list)
return return_dict
except (ValueError, TypeError) as e:
bl.debug_warning("Exception when creating new scene: {}".format(e))
return None
def setup_logging(logging_level):
"""
Setup the loggers.
"""
cl(logging_level) # setup simulation logging
cl.info("Started Core logging with level '{}'".format(logging_level))
sl(logging_level) # setup simulation logging
sl.info("Started Simulation logging with level '{}'".format(logging_level))
bl(logging_level) # setup backend logging
bl.info("Started Backend logging with level '{}'".format(logging_level))
async def _send_index_to_client(self, reader, writer, index):
"""
Prepares a dictionary with the requested index.
"""
bl.debug("Sending index to client")
todo_val = "index"
index_dictionary = {"todo": todo_val, todo_val: index}
await self._send_dictionary(reader, writer, index_dictionary)
def _unsubscribe(dataset_hash):
"""
Unsubscribe from timestep updates.
"""
with SD_LOCK:
try:
subscription = SUBSCRIPTION_DICT[dataset_hash]
except KeyError:
pass
else:
bl.debug("Setting delete flag")
subscription["delete"] = True
async def _inform_client_new_file(self, reader, writer, new_file):
"""
Prepares a dictionary with information about the new file at the ceph
cluster and sends it out via the socket connection.
"""
bl.debug("Sending information about new file to client ({})".format(
new_file))
todo_val = "new_file"
new_file_dictionary = {"todo": todo_val, todo_val: new_file}
await self._send_dictionary(reader, writer, new_file_dictionary)
def stop(self):
bl.info("Stopping BackendManager")
# await self._cancel()
try:
self._cancel_new_file_executor_event.set()
except AttributeError:
pass
try:
self._cancel_file_request_answer_executor_event.set()
except AttributeError:
pass
self._loop.call_soon_threadsafe(self._loop.close())
def delete_loaded_dataset(self, scene_hash, dataset_hash):
"""
Remove a dataset from a scene.
If all datasets are gone the scene is to be deleted.
Args:
scene_hash (str): The hash of the scene from which we want to delete
a dataset.
dataset_hash (str): The hash of the dataset we want to delete.
"""
if not isinstance(scene_hash, str):
raise TypeError('scene_hash is {}, expected str'.format(
type(scene_hash).__name__))
if not isinstance(dataset_hash, str):
raise TypeError('dataset_hash is {}, expected str'.format(
type(dataset_hash).__name__))
# If the scene does not exist
if scene_hash not in self._scene_list:
return None
target_scene = self.scene(scene_hash)
try:
remaining_datasets = target_scene.delete_dataset(dataset_hash)
except ValueError as e:
bl.debug_warning(
"Exception in delete_loaded_dataset: {}".format(e))
# The dataset does not exist
return None
# If there are no more datasets left delete the scene
if remaining_datasets == []:
self.delete_scene(scene_hash)
# We should probably return something else so we can distinguish
# between errors and deleted scenes.
return None
return_dict = {
'datasetDeleted': dataset_hash,
'href': '/scenes/{}'.format(scene_hash)
}
return return_dict
async def check_ack(self, reader):
"""
Check for ack or nack.
Returns True (ACK) or False (NACK).
"""
ck = await reader.read(8)
try:
ck = ck.decode("UTF-8")
except Exception as e:
bl.error("An Exception occured: {}".format(e))
raise
else:
if ck.lower() == "ack":
return True
return False
def init(source_dict=None):
"""
Initialise the global variables.
The scene_manager is an object that contains all the scenes on the server
and exposes methods to manipulate them and the objects that are contained.
Args:
source_dict (dict): Information about the provided data source.
Notes:
Import this module everywhere you need to manipulate scenes.
"""
bl.verbose("Creating global scene manager instance")
global scene_manager # This gets exposed
scene_manager = SceneManager(source_dict=source_dict)
async def _rw_handler(self, reader, writer):
"""
This gets called when a connection is established.
"""
connection_info = writer.get_extra_info('peername')
p_host = connection_info[0]
p_port = connection_info[1]
bl.info('Connection established from {}:{}'.format(
p_host, p_port))
self.new_file_task = self.loop.create_task(self.push_new_file(reader, writer))
self.data_index_task = self.loop.create_task(self.get_data_and_index(reader, writer))
try:
await self.new_file_task
await self.data_index_task
except Exception as e:
bl.error("Exception: {}".format(e))
finally:
bl.info('Connection terminated')
writer.close()
def _create_index_entry_from_new_file_dict(self, new_file_dict):
"""
Create a dictionary entry from the new file dictionary.
TODO: Refactor this.
"""
key = new_file_dict["key"]
namespace = new_file_dict["namespace"]
sha1sum = new_file_dict["sha1sum"]
key_dict = self._create_dict_from_key(key, sha1sum=sha1sum)
if key_dict is not None:
return_dict = dict()
return_dict[namespace] = key_dict
else:
bl.debug("Can not add file {}/{}".format(namespace, key))
return
return return_dict
def scene(self, scene_hash):
"""
Return a scene object.
Args:
scene_hash (str): The unique identifier of the scene that we want to
return.
Returns:
None or _ScenePrototype object: None if no scene with a matching hash
could be found, otherwise return the scene object.
Raises:
TypeError: If ``type(scene_hash)`` is not `str`.
See Also:
:py:class:`backend.scenes_scene_prototype._ScenePrototype`
"""
if not isinstance(scene_hash, str):
raise TypeError('scene_hash is {}, expected str'.format(
type(scene_hash).__name__))
try:
# See which index fits to the provided scene id
index = list(self._scene_list.keys()).index(scene_hash)
# Get all the scene objects out of the _scene_list
scenes = list(self._scene_list.values())
return scenes[index]
except ValueError as e:
bl.debug_warning("Scene with hash {} not found: {}".format(
scene_hash, e))
return None
def __init__(self, host, port, new_file_send_queue, get_index_server_event,
index_data_queue, file_name_request_server_queue,
file_content_name_hash_server_queue,
shutdown_backend_manager_event):
bl.info("BackendManager init: {}:{}".format(host, port))
self._host = host
self._port = port
# expose events, pipes and queues to the class
self._new_file_send_queue = new_file_send_queue
self._get_index_server_event = get_index_server_event
self._index_data_queue = index_data_queue
self._file_name_request_server_queue = file_name_request_server_queue
self._file_content_name_hash_server_queue = file_content_name_hash_server_queue
self._shutdown_backend_manager_event = shutdown_backend_manager_event
# create a server
self._loop = asyncio.get_event_loop()
self._coro = asyncio.start_server(self._rw_handler,
self._host,
self._port,
loop=self._loop,
backlog=100)
self._server = self._loop.run_until_complete(self._coro)
self._new_file_connection_active = False
self._index_connection_active = False
self._file_requests_connection_active = False
self._file_answers_connection_active = False
# download data from the ceph manager and store it in a dictionary
#
self._ceph_data_dict = dict()
# we need a threading lock and not a asyncio lock because we use them in
# an executor (extra tread)
self._ceph_data_lock = threading.Lock()
ceph_data_task = self._loop.create_task(self._ceph_data_coro())
perdiodically_delete_ceph_data_task = self._loop.create_task(
self._periodic_ceph_file_deletion_coro())
# manage the queue cleanup when there are no active connections
queue_cleanup_task = self._loop.create_task(self._queue_cleanup_coro())
shutdown_watch_task = self._loop.create_task(
self._watch_shutdown_event_coro())
bl.info("Starting BackendManager")
try:
self._loop.run_forever()
except KeyboardInterrupt:
pass # quiet KeyboardInterrupt
finally:
bl.info("BackendManager stopped")
def start(self):
try:
bl.info('Starting BackendManager on port {}'.format(self.port))
bl.info("\tConnect the platt backend to {}:{}".format(self.host, self.port))
self.loop.run_forever()
except KeyboardInterrupt:
self.stop()
finally:
bl.debug('BackendManager closed')
self.loop.close()
async def read_data(self, reader, writer):
"""
Read data from the connection.
NOTE: Do not forget to send an ACK or NACK after using this method.
Otherwise the connection might hang up.
await self.send_ack(writer)
await self.send_nack(writer)
"""
# wait until we have read something that is up to 1k (until the newline
# comes)
length_b = await reader.read(1024)
if reader.at_eof():
return
try:
# try and parse it as an int (expecting the length of the data)
length = struct.unpack("L", length_b)[0]
except Exception as e:
# if something goes wrong send a nack and start anew
await self.send_nack(writer)
bl.error("An Exception occured: {}".format(e))
raise
return
else:
# otherwise send the ack
await self.send_ack(writer)
try:
# try and read exactly the length of the data
data = await reader.readexactly(length)
res = data.decode("UTF-8")
res = json.loads(res)
except json.decoder.JSONDecodeError:
# if we can not parse the json send a nack and start from the
# beginning
bl.debug("Parsing {} as json failed".format(res))
await self.send_nack(writer)
raise
except Exception as e:
# if ANYTHING else goes wrong send a nack and start from the
# beginning
await self.send_nack(writer)
bl.error("An Exception occured: {}".format(e))
raise
else:
# otherwise return the received data
return res
