def __init__(self,
topic,
producer,
serializer=None,
timeout=0,
logger=None):
assert is_str(topic), 'topic must be a str'
assert isinstance(producer, KafkaProducer), 'bad producer instance'
assert is_none_or_func(serializer), 'serializer must be a callable'
assert is_number(timeout), 'timeout must be an int or float'
assert timeout >= 0, 'timeout must be 0 or greater'
assert is_none_or_logger(logger), 'bad logger instance'
self._topic = topic
self._hosts = producer.config['bootstrap_servers']
self._producer = producer
self._serializer = serializer or dill.dumps
self._timeout = timeout
self._logger = logger or logging.getLogger('kq.queue')
self._default_enqueue_spec = EnqueueSpec(topic=self._topic,
producer=self._producer,
serializer=self._serializer,
logger=self._logger,
timeout=self._timeout,
key=None,
partition=None)
def __init__(
self,
topic: str,
producer: KafkaProducer,
serializer: Optional[Callable[..., bytes]] = None,
timeout: int = 0,
logger: Optional[logging.Logger] = None,
) -> None:
assert is_str(topic), "topic must be a str"
assert isinstance(producer, KafkaProducer), "bad producer instance"
assert is_none_or_func(serializer), "serializer must be a callable"
assert is_number(timeout), "timeout must be an int or float"
assert timeout >= 0, "timeout must be 0 or greater"
assert is_none_or_logger(logger), "bad logger instance"
self._topic = topic
self._hosts: str = producer.config["bootstrap_servers"]
self._producer = producer
self._serializer = serializer or dill.dumps
self._timeout = timeout
self._logger = logger or logging.getLogger("kq.queue")
self._default_enqueue_spec = EnqueueSpec(
topic=self._topic,
producer=self._producer,
serializer=self._serializer,
logger=self._logger,
timeout=self._timeout,
key=None,
partition=None,
)
def enqueue(self, obj, *args, **kwargs):
"""Enqueue a function call or :doc:`job` instance.
:param func: Function or :doc:`job <job>`. Must be serializable and
importable by :doc:`worker <worker>` processes.
:type func: callable | :doc:`kq.Job <job>`
:param args: Positional arguments for the function. Ignored if **func**
is a :doc:`job <job>` object.
:param kwargs: Keyword arguments for the function. Ignored if **func**
is a :doc:`job <job>` object.
:return: Enqueued job.
:rtype: :doc:`kq.Job <job>`
"""
timestamp = int(time.time() * 1000)
if isinstance(obj, Job):
job_id = uuid.uuid4().hex if obj.id is None else obj.id
func = obj.func
args = tuple() if obj.args is None else obj.args
kwargs = {} if obj.kwargs is None else obj.kwargs
timeout = self._timeout if obj.timeout is None else obj.timeout
key = self._key if obj.key is None else obj.key
partition = self._part if obj.partition is None else obj.partition
assert is_str(job_id), 'Job.id must be a str'
assert callable(func), 'Job.func must be a callable'
assert is_iter(args), 'Job.args must be a list or tuple'
assert is_dict(kwargs), 'Job.kwargs must be a dict'
assert is_number(timeout), 'Job.timeout must be an int or float'
assert is_none_or_bytes(key), 'Job.key must be a bytes'
assert is_none_or_int(partition), 'Job.partition must be an int'
else:
assert callable(obj), 'first argument must be a callable'
job_id = uuid.uuid4().hex
func = obj
args = args
kwargs = kwargs
timeout = self._timeout
key = self._key
partition = self._part
job = Job(id=job_id,
timestamp=timestamp,
topic=self._topic,
func=func,
args=args,
kwargs=kwargs,
timeout=timeout,
key=key,
partition=partition)
self._logger.info('Enqueueing {} ...'.format(job))
self._producer.send(self._topic,
value=self._serializer(job),
key=self._serializer(key) if key else None,
partition=partition,
timestamp_ms=timestamp)
return job
def __init__(self, topic, producer, serializer, logger, timeout, key,
partition):
assert is_number(timeout), 'timeout must be an int or float'
assert is_none_or_bytes(key), 'key must be a bytes'
assert is_none_or_int(partition), 'partition must be an int'
self._topic = topic
self._producer = producer
self._serializer = serializer
self._logger = logger
self._timeout = timeout
self._key = key
self._part = partition
def __init__(
self,
topic: str,
producer: KafkaProducer,
serializer: Callable[[Any], bytes],
logger: logging.Logger,
timeout: Union[float, int],
key: Optional[bytes],
partition: Optional[int],
):
assert is_number(timeout), "timeout must be an int or float"
assert is_none_or_bytes(key), "key must be a bytes"
assert is_none_or_int(partition), "partition must be an int"
self._topic = topic
self._producer = producer
self._serializer = serializer
self._logger = logger
self._timeout = timeout
self._key = key
self._partition = partition
def enqueue(self, obj: Union[Callable[..., Any], Job], *args: Any,
**kwargs: Any) -> Job:
"""Enqueue a function call or :doc:`job` instance.
:param obj: Function or :doc:`job <job>`. Must be serializable and
importable by :doc:`worker <worker>` processes.
:type obj: callable | :doc:`kq.Job <job>`
:param args: Positional arguments for the function. Ignored if **func**
is a :doc:`job <job>` object.
:param kwargs: Keyword arguments for the function. Ignored if **func**
is a :doc:`job <job>` object.
:return: Enqueued job.
:rtype: :doc:`kq.Job <job>`
"""
timestamp = int(time.time() * 1000)
if isinstance(obj, Job):
if obj.id is None:
job_id = uuid.uuid4().hex
else:
assert is_str(obj.id), "Job.id must be a str"
job_id = obj.id
if obj.args is None:
args = tuple()
else:
assert is_seq(obj.args), "Job.args must be a list or tuple"
args = tuple(obj.args)
assert callable(obj.func), "Job.func must be a callable"
func = obj.func
kwargs = {} if obj.kwargs is None else obj.kwargs
timeout = self._timeout if obj.timeout is None else obj.timeout
key = self._key if obj.key is None else obj.key
part = self._partition if obj.partition is None else obj.partition
assert is_dict(kwargs), "Job.kwargs must be a dict"
assert is_number(timeout), "Job.timeout must be an int or float"
assert is_none_or_bytes(key), "Job.key must be a bytes"
assert is_none_or_int(part), "Job.partition must be an int"
else:
assert callable(obj), "first argument must be a callable"
job_id = uuid.uuid4().hex
func = obj
args = args
kwargs = kwargs
timeout = self._timeout
key = self._key
part = self._partition
job = Job(
id=job_id,
timestamp=timestamp,
topic=self._topic,
func=func,
args=args,
kwargs=kwargs,
timeout=timeout,
key=key,
partition=part,
)
self._logger.info(f"Enqueueing {job} ...")
self._producer.send(
self._topic,
value=self._serializer(job),
key=self._serializer(key) if key else None,
partition=part,
timestamp_ms=timestamp,
)
self._producer.flush()
return job