def _send(self, topic, kmsg, timeout=60):
""" Send the message into the given topic
:param str topic: a kafka topic
:param ksr.transport.Message kmsg: Message to serialize
:param int timeout: Timeout in seconds (not used in proto producer)
:return: Execution result
:rtype: kser.result.Result
"""
result = Result(uuid=kmsg.uuid)
try:
self.client.send(
topic, self._onmessage(kmsg).dumps().encode("UTF-8")
)
result.stdout = "Message {}[{}] sent in {}".format(
kmsg.entrypoint, kmsg.uuid, topic
)
self.client.flush()
except Exception as exc:
result = Result.from_exception(exc, kmsg.uuid)
finally:
if result.retcode < 300:
return self._onsuccess(kmsg=kmsg, result=result)
else:
return self._onerror(kmsg=kmsg, result=result)
def send(self, topic, kmsg):
""" Send the message into the given topic
:param str topic: a kafka topic
:param ksr.transport.Message kmsg: Message to serialize
:return: Execution result
:rtype: kser.result.Result
"""
try:
self.client.do_request(method="POST",
params=dict(format="raw"),
path="/topic/{}".format(topic),
data=kmsg.MARSHMALLOW_SCHEMA.dump(kmsg))
result = Result(uuid=kmsg.uuid,
stdout="Message sent: {} ({})".format(
kmsg.uuid, kmsg.entrypoint))
except Exception as exc:
result = Result.from_exception(exc, kmsg.uuid)
finally:
# noinspection PyUnboundLocalVariable
if result.retcode < 300:
return self._onsuccess(kmsg=kmsg, result=result)
else:
return self._onerror(kmsg=kmsg, result=result)
def __init__(self,
uuid,
entrypoint,
params=None,
result=None,
metadata=None):
Base.__init__(self, uuid, entrypoint, params)
if result:
if isinstance(result, Result):
self.result = result
else:
self.result = Result(**result)
else:
self.result = Result(uuid=uuid)
self.metadata = metadata or dict()
def bulk_send(self, topic, kmsgs):
""" Send a batch of messages
:param str topic: a kafka topic
:param ksr.transport.Message kmsgs: Messages to serialize
:return: Execution result
:rtype: kser.result.Result
"""
try:
self.client.do_request(
method="POST",
path="/topic/{}".format(topic),
data=[dict(Value=k.MARSHMALLOW_SCHEMA.dump(k)) for k in kmsgs])
return Result(stdout="{} message(s) sent".format(len(kmsgs)))
except Exception as exc:
return Result.from_exception(exc)
def bulk_send(self, topic, kmsgs, timeout=60):
""" Send a batch of messages
:param str topic: a kafka topic
:param ksr.transport.Message kmsgs: Messages to serialize
:param int timeout: Timeout in seconds
:return: Execution result
:rtype: kser.result.Result
"""
try:
for kmsg in kmsgs:
self.client.send(topic,
self._onmessage(kmsg).dumps().encode("UTF-8"))
self.client.flush(timeout=timeout)
return Result(stdout="{} message(s) sent".format(len(kmsgs)))
except Exception as exc:
return Result.from_exception(exc)
def execute(self, result=None):
""" Execution 'wrapper' to make sure that it return a result
:return: Execution result
:rtype: kser.result.Result
"""
try:
return self.unsafe_execute(result=result)
except Exception as exc:
return self._onerror(Result.from_exception(exc, uuid=self.uuid))
def send(self, topic, kmsg, timeout=60):
result = Result(uuid=kmsg.uuid)
try:
self.client.send(topic, CryptoSchema(context=dict(
secretbox_key=os.getenv("KSER_SECRETBOX_KEY", None)
)).encode(self._onmessage(kmsg)).encode("UTF-8"))
result.stdout = "Message {}[{}] sent in {}".format(
kmsg.entrypoint, kmsg.uuid, topic
)
self.client.flush()
except Exception as exc:
result = Result.from_exception(exc, kmsg.uuid)
finally:
if result.retcode < 300:
return self._onsuccess(kmsg=kmsg, result=result)
else:
return self._onerror(kmsg=kmsg, result=result)
def run(cls, raw_data):
"""description of run"""
logger.debug("{}.ReceivedFromKafka: {}".format(
cls.__name__, raw_data
))
try:
kmsg = cls._onmessage(cls.TRANSPORT.loads(raw_data))
except Exception as exc:
error = from_exc(exc)
logger.error(
"{}.ImportError: Failed to load data from kafka: {} "
"<- {}".format(cls.__name__, exc, raw_data),
extra=dict(kafka_raw_data=raw_data, error=error.to_dict())
)
return Result.from_error(error)
try:
cls.start_processing(kmsg)
if kmsg.entrypoint not in cls.ENTRYPOINTS:
raise ValidationError(
"Entrypoint '{}' not registred".format(kmsg.entrypoint),
extra=dict(
uuid=kmsg.uuid, entrypoint=kmsg.entrypoint,
allowed=list(cls.ENTRYPOINTS.keys())
)
)
result = cls.ENTRYPOINTS[kmsg.entrypoint].from_Message(
kmsg
).execute()
except Exception as exc:
result = Result.from_exception(exc, kmsg.uuid)
finally:
cls.stop_processing()
# noinspection PyUnboundLocalVariable
if result and result.retcode < 300:
return cls._onsuccess(kmsg=kmsg, result=result)
else:
return cls._onerror(kmsg=kmsg, result=result)
def launch_next(self, task=None, result=None):
""" Launch next task or finish operation
:param kser_operation.task.Task task: previous task
:param cdumay_result.Result result: previous task result
:return: Execution result
:rtype: cdumay_result.Result
"""
if task:
next_task = self.next(task)
if next_task:
return next_task.send(result=result)
else:
return self.set_status(task.status, result)
elif len(self.tasks) > 0:
return self.tasks[0].send(result=result)
else:
return Result(retcode=1, stderr="Nothing to do, empty operation !")
def _post_init(self):
"""A post init trigger wrapper"""
try:
return self.postinit()
except Exception as exc:
return self._onerror(Result.from_exception(exc, uuid=self.uuid))
def __init__(self, uuid=None, params=None, result=None, metadata=None):
self.uuid = uuid or str(uuid4())
self.params = params or dict()
self.result = result or Result(uuid=self.uuid)
self.metadata = metadata or dict()
self._post_init()