async def _delete_state_from_redis(self, redis_dict: RedisFlatDict,
key: str) -> None:
# Ensure that the object isn't updated before deletion
with redis_dict.lock(key):
deleted = redis_dict.delete_garbage(key)
if deleted:
logging.debug(
"Successfully garbage collected "
"state for key: %s", key)
else:
logging.debug(
"Successfully garbage collected "
"state in cloud for key %s. "
"Didn't delete locally as the "
"object is no longer garbage", key)
class GatewayDirectoryServiceRpcServicer(GatewayDirectoryServiceServicer):
"""gRPC based server for the Directoryd Gateway service"""
def __init__(self, print_grpc_payload: bool = False):
"""Initialize Directoryd grpc endpoints."""
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
self._print_grpc_payload = print_grpc_payload
if self._print_grpc_payload:
logging.info("Printing GRPC messages")
def add_to_server(self, server):
""" Add the servicer to a gRPC server """
add_GatewayDirectoryServiceServicer_to_server(self, server)
@return_void
def UpdateRecord(self, request, context):
""" Update the directory record of an object
Args:
request (UpdateRecordRequest): update record request
"""
logging.debug("UpdateRecord request received")
self._print_grpc(request)
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"UpdateRecordRequest")
return
try:
# Lock Redis for requested key until update is complete
with self._redis_dict.lock(request.id):
hwid = get_gateway_hwid()
record = self._redis_dict.get(request.id) or \
DirectoryRecord(location_history=[hwid],
identifiers={})
if record.location_history[0] != hwid:
record.location_history = [hwid] + record.location_history
for field_key in request.fields:
record.identifiers[field_key] = request.fields[field_key]
# Truncate location history to the five most recent hwid's
record.location_history = \
record.location_history[:LOCATION_MAX_LEN]
self._redis_dict[request.id] = record
except (RedisError, LockError) as e:
logging.error(e)
context.set_code(grpc.StatusCode.UNAVAILABLE)
context.set_details("Could not connect to redis: %s" % e)
@return_void
def DeleteRecord(self, request, context):
""" Delete the directory record for an ID
Args:
request (DeleteRecordRequest): delete record request
"""
logging.debug("DeleteRecord request received")
self._print_grpc(request)
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"DeleteRecordRequest")
return
# Lock Redis for requested key until delete is complete
try:
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return
self._redis_dict.mark_as_garbage(request.id)
except (RedisError, LockError) as e:
logging.error(e)
context.set_code(grpc.StatusCode.UNAVAILABLE)
context.set_details("Could not connect to redis: %s" % e)
def GetDirectoryField(self, request, context):
""" Get the directory record field for an ID and key
Args:
request (GetDirectoryFieldRequest): get directory field request
"""
logging.debug("GetDirectoryField request received")
self._print_grpc(request)
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"GetDirectoryFieldRequest")
return
if len(request.field_key) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("Field key argument cannot be empty in "
"GetDirectoryFieldRequest")
response = DirectoryField()
self._print_grpc(response)
return response
# Lock Redis for requested key until get is complete
try:
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return DirectoryField()
record = self._redis_dict[request.id]
except (RedisError, LockError) as e:
logging.error(e)
context.set_code(grpc.StatusCode.UNAVAILABLE)
context.set_details("Could not connect to redis: %s" % e)
response = DirectoryField()
self._print_grpc(response)
return response
if request.field_key not in record.identifiers:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Field %s was not found in record for "
"ID %s" % (request.field_key, request.id))
return DirectoryField()
response = DirectoryField(key=request.field_key,
value=record.identifiers[request.field_key])
self._print_grpc(response)
return response
def GetAllDirectoryRecords(self, request, context):
""" Get all directory records
Args:
request (Void): void
"""
logging.debug("GetAllDirectoryRecords request received")
self._print_grpc(request)
response = AllDirectoryRecords()
try:
redis_keys = self._redis_dict.keys()
except RedisError as e:
logging.error(e)
context.set_code(grpc.StatusCode.UNAVAILABLE)
context.set_details("Could not connect to redis: %s" % e)
self._print_grpc(request)
return response
for key in redis_keys:
try:
with self._redis_dict.lock(key):
# Lookup may produce an exception if the key has been
# deleted between the call to __iter__ and lock
stored_record = self._redis_dict[key]
except (RedisError, LockError) as e:
logging.error(e)
context.set_code(grpc.StatusCode.UNAVAILABLE)
context.set_details("Could not connect to redis: %s" % e)
self._print_grpc(response)
return response
except KeyError:
continue
directory_record = response.records.add()
directory_record.id = key
directory_record.location_history[:] = \
stored_record.location_history
for identifier_key in stored_record.identifiers:
directory_record.fields[identifier_key] = \
stored_record.identifiers[identifier_key]
self._print_grpc(response)
return response
def _print_grpc(self, message):
if self._print_grpc_payload:
log_msg = "{} {}".format(message.DESCRIPTOR.full_name,
MessageToJson(message))
# add indentation
padding = 2 * ' '
log_msg = ''.join("{}{}".format(padding, line)
for line in log_msg.splitlines(True))
log_msg = "GRPC message:\n{}".format(log_msg)
logging.info(log_msg)
class GatewayDirectoryServiceRpcServicer(GatewayDirectoryServiceServicer):
""" gRPC based server for the Directoryd Gateway service. """
def __init__(self):
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
def add_to_server(self, server):
""" Add the servicer to a gRPC server """
add_GatewayDirectoryServiceServicer_to_server(self, server)
@return_void
def UpdateRecord(self, request, context):
""" Update the directory record of an object
Args:
request (UpdateRecordRequest): update record request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"UpdateRecordRequest")
return
# Lock Redis for requested key until update is complete
with self._redis_dict.lock(request.id):
hwid = get_gateway_hwid()
record = self._redis_dict.get(request.id) or \
DirectoryRecord(location_history=[hwid], identifiers={})
if record.location_history[0] != hwid:
record.location_history = [hwid] + record.location_history
for field_key in request.fields:
record.identifiers[field_key] = request.fields[field_key]
# Truncate location history to the five most recent hwid's
record.location_history = \
record.location_history[:LOCATION_MAX_LEN]
self._redis_dict[request.id] = record
@return_void
def DeleteRecord(self, request, context):
""" Delete the directory record for an ID
Args:
request (DeleteRecordRequest): delete record request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"DeleteRecordRequest")
return
# Lock Redis for requested key until delete is complete
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return
self._redis_dict.mark_as_garbage(request.id)
def GetDirectoryField(self, request, context):
""" Get the directory record field for an ID and key
Args:
request (GetDirectoryFieldRequest): get directory field request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"GetDirectoryFieldRequest")
return
if len(request.field_key) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("Field key argument cannot be empty in "
"GetDirectoryFieldRequest")
return
# Lock Redis for requested key until get is complete
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return
record = self._redis_dict[request.id]
if request.field_key not in record.identifiers:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Field %s was not found in record for "
"ID %s" % (request.field_key, request.id))
return
return DirectoryField(key=request.field_key,
value=record.identifiers[request.field_key])
def GetAllDirectoryRecords(self, request, context):
""" Get all directory records
Args:
request (Void): void
"""
response = AllDirectoryRecords()
for key in self._redis_dict.keys():
with self._redis_dict.lock(key):
# Lookup may produce an exception if the key has been deleted
# between the call to __iter__ and lock
try:
stored_record = self._redis_dict[key]
except KeyError:
continue
directory_record = response.records.add()
directory_record.id = key
directory_record.location_history[:] = \
stored_record.location_history
for identifier_key in stored_record.identifiers:
directory_record.fields[identifier_key] = \
stored_record.identifiers[identifier_key]
return response
class GatewayDirectoryServiceRpcServicer(GatewayDirectoryServiceServicer):
""" gRPC based server for the Directoryd Gateway service. """
def __init__(self):
serde = RedisSerde(DIRECTORYD_REDIS_TYPE, get_json_serializer(),
get_json_deserializer())
self._redis_dict = RedisFlatDict(get_default_client(), serde)
def add_to_server(self, server):
""" Add the servicer to a gRPC server """
add_GatewayDirectoryServiceServicer_to_server(self, server)
@return_void
def UpdateRecord(self, request, context):
""" Update the directory record of an object
Args:
request (UpdateRecordRequest): update record request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"UpdateRecordRequest")
return
# Lock Redis for requested key until update is complete
with self._redis_dict.lock(request.id):
hwid = get_gateway_hwid()
record = self._redis_dict.get(request.id) or \
DirectoryRecord(location_history=[hwid], identifiers={})
if record.location_history[0] != hwid:
record.location_history = [hwid] + record.location_history
for field in request.fields:
record.identifiers[field.key] = field.value
# Truncate location history to the five most recent hwid's
record.location_history = \
record.location_history[:LOCATION_MAX_LEN]
self._redis_dict[request.id] = record
@return_void
def DeleteRecord(self, request, context):
""" Delete the directory record for an ID
Args:
request (DeleteRecordRequest): delete record request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"DeleteRecordRequest")
return
# Lock Redis for requested key until delete is complete
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return
# TODO: Set record to be garbage collected rather than deleting
# directly
del self._redis_dict[request.id]
def GetDirectoryField(self, request, context):
""" Get the directory record field for an ID and key
Args:
request (GetDirectoryFieldRequest): get directory field request
"""
if len(request.id) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("ID argument cannot be empty in "
"GetDirectoryFieldRequest")
return
if len(request.field_key) == 0:
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
context.set_details("Field key argument cannot be empty in "
"GetDirectoryFieldRequest")
return
# Lock Redis for requested key until get is complete
with self._redis_dict.lock(request.id):
if request.id not in self._redis_dict:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Record for ID %s was not found." %
request.id)
return
record = self._redis_dict[request.id]
if request.field_key not in record.identifiers:
context.set_code(grpc.StatusCode.NOT_FOUND)
context.set_details("Field %s was not found in record for "
"ID %s" % (request.field_key, request.id))
return
return DirectoryField(key=request.field_key,
value=record.identifiers[request.field_key])