def get_service_binds(config: Config) -> List[KartonBind]:
redis = StrictRedis(decode_responses=True, **config.redis_config)
replica_no: Dict[Any, int] = Counter()
# count replicas for each identity
for client in redis.client_list():
replica_no[client["name"]] += 1
binds = redis.hgetall("karton.binds")
services = []
for identity, data in binds.items():
val = json.loads(data)
# karton 2.x compatibility :(
if isinstance(val, list):
val = val[0]
services.append(
KartonBind(
identity,
replica_no[identity],
val.get("persistent", True),
val.get("version", "2.x.x"),
val.get("filters", []),
))
return services
class KartonBackend:
def __init__(self, config):
self.config = config
self.redis = StrictRedis(
host=config["redis"]["host"],
port=int(config["redis"].get("port", 6379)),
decode_responses=True,
)
self.minio = Minio(
config["minio"]["address"],
access_key=config["minio"]["access_key"],
secret_key=config["minio"]["secret_key"],
secure=bool(int(config["minio"].get("secure", True))),
)
@property
def default_bucket_name(self) -> str:
return self.config.minio_config["bucket"]
@staticmethod
def get_queue_name(identity: str, priority: TaskPriority) -> str:
"""
Return Redis routed task queue name for given identity and priority
:param identity: Karton service identity
:param priority: Queue priority (TaskPriority enum value)
:return: Queue name
"""
return f"karton.queue.{priority.value}:{identity}"
@staticmethod
def get_queue_names(identity: str) -> List[str]:
"""
Return all Redis routed task queue names for given identity,
ordered by priority (descending). Used internally by Consumer.
:param identity: Karton service identity
:return: List of queue names
"""
return [
identity, # Backwards compatibility (2.x.x)
KartonBackend.get_queue_name(identity, TaskPriority.HIGH),
KartonBackend.get_queue_name(identity, TaskPriority.NORMAL),
KartonBackend.get_queue_name(identity, TaskPriority.LOW),
]
@staticmethod
def serialize_bind(bind: KartonBind) -> str:
"""
Serialize KartonBind object (Karton service registration)
:param bind: KartonBind object with bind definition
:return: Serialized bind data
"""
return json.dumps(
{
"info": bind.info,
"version": bind.version,
"filters": bind.filters,
"persistent": bind.persistent,
},
sort_keys=True,
)
@staticmethod
def unserialize_bind(identity: str, bind_data: str) -> KartonBind:
"""
Deserialize KartonBind object for given identity.
Compatible with Karton 2.x.x and 3.x.x
:param identity: Karton service identity
:param bind_data: Serialized bind data
:return: KartonBind object with bind definition
"""
bind = json.loads(bind_data)
if isinstance(bind, list):
# Backwards compatibility (v2.x.x)
return KartonBind(
identity=identity,
info=None,
version="2.x.x",
persistent=not identity.endswith(".test"),
filters=bind,
)
return KartonBind(
identity=identity,
info=bind["info"],
version=bind["version"],
persistent=bind["persistent"],
filters=bind["filters"],
)
def get_bind(self, identity: str) -> KartonBind:
"""
Get bind object for given identity
:param identity: Karton service identity
:return: KartonBind object
"""
return self.unserialize_bind(
identity, self.redis.hget(KARTON_BINDS_HSET, identity))
def get_binds(self) -> List[KartonBind]:
"""
Get all binds registered in Redis
:return: List of KartonBind objects for subsequent identities
"""
return [
self.unserialize_bind(identity, raw_bind) for identity, raw_bind in
self.redis.hgetall(KARTON_BINDS_HSET).items()
]
def register_bind(self, bind: KartonBind) -> Optional[KartonBind]:
"""
Register bind for Karton service and return the old one
:param bind: KartonBind object with bind definition
:return: Old KartonBind that was registered under this identity
"""
with self.redis.pipeline(transaction=True) as pipe:
pipe.hget(KARTON_BINDS_HSET, bind.identity)
pipe.hset(KARTON_BINDS_HSET, bind.identity,
self.serialize_bind(bind))
old_serialized_bind, _ = pipe.execute()
if old_serialized_bind:
return self.unserialize_bind(bind.identity, old_serialized_bind)
else:
return None
def unregister_bind(self, identity: str) -> None:
"""
Removes bind for identity
:param bind: Identity to be unregistered
"""
self.redis.hdel(KARTON_BINDS_HSET, identity)
def set_consumer_identity(self, identity: str) -> None:
"""
Sets identity for current Redis connection
"""
return self.redis.client_setname(identity)
def get_online_consumers(self) -> Dict[str, List[str]]:
"""
Gets all online consumer identities
:return: Dictionary {identity: [list of clients]}
"""
bound_identities = defaultdict(list)
for client in self.redis.client_list():
bound_identities[client["name"]].append(client)
return bound_identities
def get_task(self, task_uid: str) -> Optional[Task]:
"""
Get task object with given identifier
:param task_uid: Task identifier
:return: Task object
"""
task_data = self.redis.get(f"{KARTON_TASK_NAMESPACE}:{task_uid}")
if not task_data:
return None
return Task.unserialize(task_data, backend=self)
def get_tasks(self, task_uid_list: List[str]) -> List[Task]:
"""
Get multiple tasks for given identifier list
:param task_uid_list: List of task identifiers
:return: List of task objects
"""
task_list = self.redis.mget([
f"{KARTON_TASK_NAMESPACE}:{task_uid}" for task_uid in task_uid_list
])
return [
Task.unserialize(task_data, backend=self)
for task_data in task_list if task_data is not None
]
def get_all_tasks(self) -> List[Task]:
"""
Get all tasks registered in Redis
:return: List with Task objects
"""
tasks = self.redis.keys(f"{KARTON_TASK_NAMESPACE}:*")
return [
Task.unserialize(task_data) for task_data in self.redis.mget(tasks)
if task_data is not None
]
def register_task(self, task: Task) -> None:
"""
Register task in Redis.
Consumer should register only Declared tasks.
Status change should be done using set_task_status.
:param task: Task object
"""
self.redis.set(f"{KARTON_TASK_NAMESPACE}:{task.uid}", task.serialize())
def set_task_status(self,
task: Task,
status: TaskState,
consumer: Optional[str] = None) -> None:
"""
Request task status change to be applied by karton-system
:param task: Task object
:param status: New task status (TaskState)
:param consumer: Consumer identity
"""
self.redis.rpush(
KARTON_OPERATIONS_QUEUE,
json.dumps({
"status": status.value,
"identity": consumer,
"task": task.serialize(),
"type": "operation",
}),
)
def delete_task(self, task: Task) -> None:
"""
Remove task from Redis
:param task: Task object
"""
self.redis.delete(f"{KARTON_TASK_NAMESPACE}:{task.uid}")
def get_task_queue(self, queue: str) -> List[Task]:
"""
Return all tasks in provided queue
:param queue: Queue name
:return: List with Task objects contained in queue
"""
task_uids = self.redis.lrange(queue, 0, -1)
return self.get_tasks(task_uids)
def get_task_ids_from_queue(self, queue: str) -> List[str]:
"""
Return all task UIDs in a queue
:param queue: Queue name
:return: List with task identifiers contained in queue
"""
return self.redis.lrange(queue, 0, -1)
def remove_task_queue(self, queue: str) -> List[Task]:
"""
Remove task queue with all contained tasks
:param queue: Queue name
:return: List with Task objects contained in queue
"""
pipe = self.redis.pipeline()
pipe.lrange(queue, 0, -1)
pipe.delete(queue)
return self.get_tasks(pipe.execute()[0])
def produce_unrouted_task(self, task: Task) -> None:
"""
Add given task to unrouted task (``karton.tasks``) queue
Task must be registered before with :py:meth:`register_task`
:param task: Task object
"""
self.redis.rpush(KARTON_TASKS_QUEUE, task.uid)
def produce_routed_task(self, identity: str, task: Task) -> None:
"""
Add given task to routed task queue of given identity
Task must be registered using :py:meth:`register_task`
:param identity: Karton service identity
:param task: Task object
"""
self.redis.rpush(self.get_queue_name(identity, task.priority),
task.uid)
def consume_queues(self,
queues: Union[str, List[str]],
timeout: int = 0) -> Optional[Tuple[str, str]]:
"""
Get item from queues (ordered from the most to the least prioritized)
If there are no items, wait until one appear.
:param queues: Redis queue name or list of names
:param timeout: Waiting for item timeout (default: 0 = wait forever)
:return: Tuple of [queue_name, item] objects or None if timeout has been reached
"""
return self.redis.blpop(queues, timeout=timeout)
def consume_routed_task(self,
identity: str,
timeout: int = 5) -> Optional[Task]:
"""
Get routed task for given consumer identity.
If there are no tasks, blocks until new one appears or timeout is reached.
:param identity: Karton service identity
:param timeout: Waiting for task timeout (default: 5)
:return: Task object
"""
item = self.consume_queues(
self.get_queue_names(identity),
timeout=timeout,
)
if not item:
return None
queue, data = item
return self.get_task(data)
@staticmethod
def _log_channel(logger_name: Optional[str], level: Optional[str]) -> str:
return ".".join(
[KARTON_LOG_CHANNEL, (level or "*").lower(), logger_name or "*"])
def produce_log(
self,
log_record: Dict[str, Any],
logger_name: str,
level: str,
) -> bool:
"""
Push new log record to the logs channel
:param log_record: Dict with log record
:param logger_name: Logger name
:param level: Log level
:return: True if any active log consumer received log record
"""
return (self.redis.publish(self._log_channel(logger_name, level),
json.dumps(log_record)) > 0)
def consume_log(
self,
timeout: int = 5,
logger_filter: Optional[str] = None,
level: Optional[str] = None,
) -> Iterator[Optional[Dict[str, Any]]]:
"""
Subscribe to logs channel and yield subsequent log records
or None if timeout has been reached.
If you want to subscribe only to a specific logger name
and/or log level, pass them via logger_filter and level arguments.
:param timeout: Waiting for log record timeout (default: 5)
:param logger_filter: Filter for name of consumed logger
:param level: Log level
:return: Dict with log record
"""
with self.redis.pubsub() as pubsub:
pubsub.psubscribe(self._log_channel(logger_filter, level))
while pubsub.subscribed:
item = pubsub.get_message(ignore_subscribe_messages=True,
timeout=timeout)
if item and item["type"] == "pmessage":
body = json.loads(item["data"])
if "task" in body and isinstance(body["task"], str):
body["task"] = json.loads(body["task"])
yield body
yield None
def increment_metrics(self, metric: KartonMetrics, identity: str) -> None:
"""
Increments metrics for given operation type and identity
:param metric: Operation metric type
:param identity: Related Karton service identity
"""
self.redis.hincrby(metric.value, identity, 1)
def upload_object(
self,
bucket: str,
object_uid: str,
content: Union[bytes, BinaryIO],
length: int = None,
) -> None:
"""
Upload resource object to underlying object storage (Minio)
:param bucket: Bucket name
:param object_uid: Object identifier
:param content: Object content as bytes or file-like stream
:param length: Object content length (if file-like object provided)
"""
if isinstance(content, bytes):
length = len(content)
content = BytesIO(content)
self.minio.put_object(bucket, object_uid, content, length)
def upload_object_from_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Upload resource object file to underlying object storage
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Path to the object content
"""
self.minio.fput_object(bucket, object_uid, path)
def get_object(self, bucket: str, object_uid: str) -> HTTPResponse:
"""
Get resource object stream with the content.
Returned response should be closed after use to release network resources.
To reuse the connection, it's required to call `response.release_conn()`
explicitly.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Response object with content
"""
return self.minio.get_object(bucket, object_uid)
def download_object(self, bucket: str, object_uid: str) -> bytes:
"""
Download resource object from object storage.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Content bytes
"""
reader = self.minio.get_object(bucket, object_uid)
try:
return reader.read()
finally:
reader.release_conn()
reader.close()
def download_object_to_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Download resource object from object storage to file
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Target file path
"""
self.minio.fget_object(bucket, object_uid, path)
def list_objects(self, bucket: str) -> List[str]:
"""
List identifiers of stored resource objects
:param bucket: Bucket name
:return: List of object identifiers
"""
return [
object.object_name for object in self.minio.list_objects(bucket)
]
def remove_object(self, bucket: str, object_uid: str) -> None:
"""
Remove resource object from object storage
:param bucket: Bucket name
:param object_uid: Object identifier
"""
self.minio.remove_object(bucket, object_uid)
def check_bucket_exists(self, bucket: str, create: bool = False) -> bool:
"""
Check if bucket exists and optionally create it if it doesn't.
:param bucket: Bucket name
:param create: Create bucket if doesn't exist
:return: True if bucket exists yet
"""
if self.minio.bucket_exists(bucket):
return True
if create:
self.minio.make_bucket(bucket)
return False
class Storage:
def __init__(self, bot, config_prefix):
"""
Create new storage for bot
:param bot: Leonard object
:param config_prefix: prefix of variables in config. Default - 'LEONARD_'
:return:
"""
# Connect to Redis.
# If we had problems with Redis - just set self.redis to None.
# Not redis-required modules must work without Redis.
# We get parameters for redis connection from bot's config.
self.redis = StrictRedis(
host=bot.config.get('{}REDIS_HOST'.format(config_prefix), 'localhost'),
port=bot.config.get('{}REDIS_PORT'.format(config_prefix), '6379'),
db=bot.config.get('{}REDIS_DB'.format(config_prefix), '0')
)
try:
# Check Redis connection
self.redis.client_list()
except Exception as error:
logger.error_message('Error while connecting Redis:')
logger.error_message(str(error))
self.redis = None
def get(self, key, default_value=None):
"""
Get value from redis storage
:param key: string, redis key for needed value
:param default_value: string, value that returns if
key not found or redis isn't
connected to this bot
:return: value with that key or default value
"""
# If we had problems with redis connection,
# return None or other default value
if not self.redis:
logger.warning_message("Redis not available for {} key".format(
key
))
return default_value
value = self.redis.get(key)
if value is not None:
return value
else:
return default_value
def set(self, key, value):
"""
Set key to value in redis storage
:param key: string
:param value: string
:return:
"""
if not self.redis:
logger.warning_message("{} key didn't save in storage".format(
key
))
return None
return self.redis.set(key, value)
def get_json(self, key, default_value=None):
"""
Get value from Redis and parse JSON in it
:param key: str
:param default_value: string, value that returns if
key not found or redis isn't
connected to this bot
:return: list/dict
"""
value = self.get(key, default_value)
if not value:
return value
if type(value) not in [list, dict]:
return json.loads(value.decode('utf-8'))
return value
def set_json(self, key, value):
"""
Dump value into json and save it in Redis
:param key: string
:param value: list/dict
:return:
"""
json_value = json.dumps(value)
return self.set(key, json_value)
class Storage:
def __init__(self, bot):
"""
Create new storage for bot
:param bot: Bot object
:return:
"""
self.bot = bot
# Connect to Redis.
# If we had problems with Redis - just set self.redis to None.
# Not redis-required modules must work without Redis.
# We get parameters for redis connection from bot's config.
self.redis = StrictRedis(
host=bot.config.get('SHELDON_REDIS_HOST', 'localhost'),
port=bot.config.get('SHELDON_REDIS_PORT', '6379'),
db=bot.config.get('SHELDON_REDIS_DB', '0')
)
try:
# Check Redis connection
self.redis.client_list()
except Exception as error:
logger.error_message('Error while connecting Redis:')
logger.error_message(str(error))
self.redis = None
def get(self, key, default_value=None):
"""
Get value from redis storage
:param key: string, redis key for needed value
:param default_value: string, value that returns if
key not found or redis isn't
connected to this bot
:return: value with that key or default value
"""
# If we had problems with redis connection,
# return None or other default value
if not self.redis:
logger.warning_message("Redis not available for {} key".format(
key
))
return default_value
value = self.redis.get(key)
if value is not None:
return value
else:
return default_value
def set(self, key, value):
"""
Set key to value in redis storage
:param key: string
:param value: string
:return:
"""
if not self.redis:
logger.warning_message("{} key didn't save in storage".format(
key
))
return None
return self.redis.set(key, value)
