def __init__(self, topic, producer, serializer=None, timeout=0, logger=None): assert is_str(topic), 'topic must be a str' assert isinstance(producer, KafkaProducer), 'bad producer instance' assert is_none_or_func(serializer), 'serializer must be a callable' assert is_number(timeout), 'timeout must be an int or float' assert timeout >= 0, 'timeout must be 0 or greater' assert is_none_or_logger(logger), 'bad logger instance' self._topic = topic self._hosts = producer.config['bootstrap_servers'] self._producer = producer self._serializer = serializer or dill.dumps self._timeout = timeout self._logger = logger or logging.getLogger('kq.queue') self._default_enqueue_spec = EnqueueSpec(topic=self._topic, producer=self._producer, serializer=self._serializer, logger=self._logger, timeout=self._timeout, key=None, partition=None)
def __init__( self, topic: str, producer: KafkaProducer, serializer: Optional[Callable[..., bytes]] = None, timeout: int = 0, logger: Optional[logging.Logger] = None, ) -> None: assert is_str(topic), "topic must be a str" assert isinstance(producer, KafkaProducer), "bad producer instance" assert is_none_or_func(serializer), "serializer must be a callable" assert is_number(timeout), "timeout must be an int or float" assert timeout >= 0, "timeout must be 0 or greater" assert is_none_or_logger(logger), "bad logger instance" self._topic = topic self._hosts: str = producer.config["bootstrap_servers"] self._producer = producer self._serializer = serializer or dill.dumps self._timeout = timeout self._logger = logger or logging.getLogger("kq.queue") self._default_enqueue_spec = EnqueueSpec( topic=self._topic, producer=self._producer, serializer=self._serializer, logger=self._logger, timeout=self._timeout, key=None, partition=None, )
def enqueue(self, obj, *args, **kwargs): """Enqueue a function call or :doc:`job` instance. :param func: Function or :doc:`job <job>`. Must be serializable and importable by :doc:`worker <worker>` processes. :type func: callable | :doc:`kq.Job <job>` :param args: Positional arguments for the function. Ignored if **func** is a :doc:`job <job>` object. :param kwargs: Keyword arguments for the function. Ignored if **func** is a :doc:`job <job>` object. :return: Enqueued job. :rtype: :doc:`kq.Job <job>` """ timestamp = int(time.time() * 1000) if isinstance(obj, Job): job_id = uuid.uuid4().hex if obj.id is None else obj.id func = obj.func args = tuple() if obj.args is None else obj.args kwargs = {} if obj.kwargs is None else obj.kwargs timeout = self._timeout if obj.timeout is None else obj.timeout key = self._key if obj.key is None else obj.key partition = self._part if obj.partition is None else obj.partition assert is_str(job_id), 'Job.id must be a str' assert callable(func), 'Job.func must be a callable' assert is_iter(args), 'Job.args must be a list or tuple' assert is_dict(kwargs), 'Job.kwargs must be a dict' assert is_number(timeout), 'Job.timeout must be an int or float' assert is_none_or_bytes(key), 'Job.key must be a bytes' assert is_none_or_int(partition), 'Job.partition must be an int' else: assert callable(obj), 'first argument must be a callable' job_id = uuid.uuid4().hex func = obj args = args kwargs = kwargs timeout = self._timeout key = self._key partition = self._part job = Job(id=job_id, timestamp=timestamp, topic=self._topic, func=func, args=args, kwargs=kwargs, timeout=timeout, key=key, partition=partition) self._logger.info('Enqueueing {} ...'.format(job)) self._producer.send(self._topic, value=self._serializer(job), key=self._serializer(key) if key else None, partition=partition, timestamp_ms=timestamp) return job
def __init__(self, topic, producer, serializer, logger, timeout, key, partition): assert is_number(timeout), 'timeout must be an int or float' assert is_none_or_bytes(key), 'key must be a bytes' assert is_none_or_int(partition), 'partition must be an int' self._topic = topic self._producer = producer self._serializer = serializer self._logger = logger self._timeout = timeout self._key = key self._part = partition
def __init__( self, topic: str, producer: KafkaProducer, serializer: Callable[[Any], bytes], logger: logging.Logger, timeout: Union[float, int], key: Optional[bytes], partition: Optional[int], ): assert is_number(timeout), "timeout must be an int or float" assert is_none_or_bytes(key), "key must be a bytes" assert is_none_or_int(partition), "partition must be an int" self._topic = topic self._producer = producer self._serializer = serializer self._logger = logger self._timeout = timeout self._key = key self._partition = partition
def enqueue(self, obj: Union[Callable[..., Any], Job], *args: Any, **kwargs: Any) -> Job: """Enqueue a function call or :doc:`job` instance. :param obj: Function or :doc:`job <job>`. Must be serializable and importable by :doc:`worker <worker>` processes. :type obj: callable | :doc:`kq.Job <job>` :param args: Positional arguments for the function. Ignored if **func** is a :doc:`job <job>` object. :param kwargs: Keyword arguments for the function. Ignored if **func** is a :doc:`job <job>` object. :return: Enqueued job. :rtype: :doc:`kq.Job <job>` """ timestamp = int(time.time() * 1000) if isinstance(obj, Job): if obj.id is None: job_id = uuid.uuid4().hex else: assert is_str(obj.id), "Job.id must be a str" job_id = obj.id if obj.args is None: args = tuple() else: assert is_seq(obj.args), "Job.args must be a list or tuple" args = tuple(obj.args) assert callable(obj.func), "Job.func must be a callable" func = obj.func kwargs = {} if obj.kwargs is None else obj.kwargs timeout = self._timeout if obj.timeout is None else obj.timeout key = self._key if obj.key is None else obj.key part = self._partition if obj.partition is None else obj.partition assert is_dict(kwargs), "Job.kwargs must be a dict" assert is_number(timeout), "Job.timeout must be an int or float" assert is_none_or_bytes(key), "Job.key must be a bytes" assert is_none_or_int(part), "Job.partition must be an int" else: assert callable(obj), "first argument must be a callable" job_id = uuid.uuid4().hex func = obj args = args kwargs = kwargs timeout = self._timeout key = self._key part = self._partition job = Job( id=job_id, timestamp=timestamp, topic=self._topic, func=func, args=args, kwargs=kwargs, timeout=timeout, key=key, partition=part, ) self._logger.info(f"Enqueueing {job} ...") self._producer.send( self._topic, value=self._serializer(job), key=self._serializer(key) if key else None, partition=part, timestamp_ms=timestamp, ) self._producer.flush() return job