class GearmanIn(Actor):
'''**Consumes events/jobs from Gearmand.**
Consumes jobs from a Gearmand server.
When secret is none, no decryption is done.
Parameters:
- hostlist(list)(["localhost:4730"])
| A list of gearmand servers. Each entry should have
| format host:port.
- secret(str)(None)
| The AES encryption key to decrypt Mod_gearman messages.
- workers(int)(1)
| The number of gearman workers within 1 process.
- queue(str)(wishbone)
| The queue to consume jobs from.
- enable_keepalive(bool)(False)
| Attempt to monkey patch the gearmand module to enable socket
| keepalive.
Queues:
- outbox: Outgoing events.
'''
def __init__(self,
actor_config,
hostlist=["localhost:4730"],
secret=None,
workers=1,
queue="wishbone",
enable_keepalive=False):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.background_instances = []
if self.kwargs.secret is None:
self.decrypt = self.__plainTextJob
else:
key = self.kwargs.secret[0:32]
self.cipher = AES.new(key + chr(0) * (32 - len(key)))
self.decrypt = self.__encryptedJob
def preHook(self):
if self.kwargs.enable_keepalive:
self.logging.info("Requested to monkey patch Gearmand")
if gearman_version == "2.0.2":
self.logging.info(
"Detected gearman version 2.0.2, patching sockets with SO_KEEPALIVE enabled."
)
self.gearmanWorker = self._gearmanWorkerPatched
else:
self.logging.warning(
"Did not detect gearman version 2.0.2. Not patching , patching sockets with keepalive enabled."
)
self.gearmanWorker = self._gearmanWorkerNotPatched
else:
self.gearmanWorker = self._gearmanWorkerNotPatched
for _ in range(self.kwargs.workers):
self.sendToBackground(self.gearmanWorker)
self.sendToBackground(self.monitor)
def consume(self, gearman_worker, gearman_job):
decrypted = self.decrypt(gearman_job.data)
event = Event(decrypted)
self.submit(event, self.pool.queue.outbox)
return gearman_job.data
def __encryptedJob(self, data):
return self.cipher.decrypt(base64.b64decode(data))
def __plainTextJob(self, data):
return data
def _gearmanWorkerPatched(self):
self.logging.info("Gearmand worker instance started")
while self.loop():
try:
with mock.patch.object(GearmanConnection,
'_create_client_socket',
create_client_socket):
self.worker_instance = GearmanWorker(self.kwargs.hostlist)
self.worker_instance.register_task(self.kwargs.queue,
self.consume)
self.worker_instance.work()
except Exception as err:
self.logging.warn(
"Connection to gearmand failed. Reason: '%s'. Retry in 1 second."
% err)
sleep(1)
finally:
self.worker_instance.shutdown()
def _gearmanWorkerNotPatched(self):
self.logging.info("Gearmand worker instance started")
while self.loop():
try:
self.worker_instance = GearmanWorker(self.kwargs.hostlist)
self.worker_instance.register_task(self.kwargs.queue,
self.consume)
self.worker_instance.work()
except Exception as err:
self.logging.warn(
"Connection to gearmand failed. Reason: '%s'. Retry in 1 second."
% err)
sleep(1)
finally:
self.worker_instance.shutdown()
def monitor(self):
self.logging.info("Connection monitor started.")
while self.loop():
sleep(5)
for conn in self.worker_instance.connection_list:
if not conn.connected:
self.logging.error(
"Connection to '%s' is dead. Trying to reconnect." %
(conn.gearman_host))
try:
conn.connect()
self.logging.info("Connection to '%s' is restored." %
(conn.gearman_host))
except Exception as err:
self.logging.error(
"Failed to reconnect to '%s'. Retry in 5 seconds. Reason: '%s'"
% (conn.gearman_host, err))
else:
self.logging.debug("Connection to '%s' is alive." %
(conn.gearman_host))
return False
def task_listener(gearman_worker, gearman_job):
task_name, video_id, segment_id = pickle.loads(gearman_job.data)
result = False
if task_name == 'transcode':
result = transcode_segment(video_id, segment_id)
elif task_name == 'thumbnail':
result = generate_thumbnail(video_id, segment_id)
return pickle.dumps(result)
if __name__ == "__main__":
# worker run
logger.info("Setting up the worker.")
gm_worker = GearmanWorker([GEARMAND_HOST_PORT])
gm_worker.register_task(SEGMENT_TASK_NAME, task_listener)
try:
logger.info("Worker was set up successfully. Waiting for work.")
gm_worker.work()
except KeyboardInterrupt:
gm_worker.shutdown()
logger.info("Worker has shut down successfully. Bye.")
return False
def task_listener(gearman_worker, gearman_job):
task_name, video_id, segment_id = pickle.loads(gearman_job.data)
result = False
if task_name == 'transcode':
result = transcode_segment(video_id, segment_id)
elif task_name == 'thumbnail':
result = generate_thumbnail(video_id, segment_id)
return pickle.dumps(result)
if __name__ == "__main__":
# worker run
logger.info("Setting up the worker.")
gm_worker = GearmanWorker([GEARMAND_HOST_PORT])
gm_worker.register_task(SEGMENT_TASK_NAME, task_listener)
try:
logger.info("Worker was set up successfully. Waiting for work.")
gm_worker.work()
except KeyboardInterrupt:
gm_worker.shutdown()
logger.info("Worker has shut down successfully. Bye.")
class GearmanIn(Actor):
'''**Consumes events/jobs from Gearmand.**
Consumes jobs from a Gearmand server.
When secret is none, no decryption is done.
Parameters:
- hostlist(list)(["localhost:4730"])
| A list of gearmand servers. Each entry should have
| format host:port.
- secret(str)(None)
| The AES encryption key to decrypt Mod_gearman messages.
- workers(int)(1)
| The number of gearman workers within 1 process.
- queue(str)(wishbone)
| The queue to consume jobs from.
- enable_keepalive(bool)(False)
| Attempt to monkey patch the gearmand module to enable socket
| keepalive.
Queues:
- outbox: Outgoing events.
'''
def __init__(self, actor_config, hostlist=["localhost:4730"], secret=None, workers=1, queue="wishbone", enable_keepalive=False):
Actor.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.background_instances = []
if self.kwargs.secret is None:
self.decrypt = self.__plainTextJob
else:
key = self.kwargs.secret[0:32]
self.cipher = AES.new(key + chr(0) * (32 - len(key)))
self.decrypt = self.__encryptedJob
def preHook(self):
if self.kwargs.enable_keepalive:
self.logging.info("Requested to monkey patch Gearmand")
if gearman_version == "2.0.2":
self.logging.info("Detected gearman version 2.0.2, patching sockets with SO_KEEPALIVE enabled.")
self.gearmanWorker = self._gearmanWorkerPatched
else:
self.logging.warning("Did not detect gearman version 2.0.2. Not patching , patching sockets with keepalive enabled.")
self.gearmanWorker = self._gearmanWorkerNotPatched
else:
self.gearmanWorker = self._gearmanWorkerNotPatched
for _ in range(self.kwargs.workers):
self.sendToBackground(self.gearmanWorker)
self.sendToBackground(self.monitor)
def consume(self, gearman_worker, gearman_job):
decrypted = self.decrypt(gearman_job.data)
event = Event(decrypted)
self.submit(event, self.pool.queue.outbox)
return gearman_job.data
def __encryptedJob(self, data):
return self.cipher.decrypt(base64.b64decode(data))
def __plainTextJob(self, data):
return data
def _gearmanWorkerPatched(self):
self.logging.info("Gearmand worker instance started")
while self.loop():
try:
with mock.patch.object(GearmanConnection, '_create_client_socket', create_client_socket):
self.worker_instance = GearmanWorker(self.kwargs.hostlist)
self.worker_instance.register_task(self.kwargs.queue, self.consume)
self.worker_instance.work()
except Exception as err:
self.logging.warn("Connection to gearmand failed. Reason: '%s'. Retry in 1 second." % err)
sleep(1)
finally:
self.worker_instance.shutdown()
def _gearmanWorkerNotPatched(self):
self.logging.info("Gearmand worker instance started")
while self.loop():
try:
self.worker_instance = GearmanWorker(self.kwargs.hostlist)
self.worker_instance.register_task(self.kwargs.queue, self.consume)
self.worker_instance.work()
except Exception as err:
self.logging.warn("Connection to gearmand failed. Reason: '%s'. Retry in 1 second." % err)
sleep(1)
finally:
self.worker_instance.shutdown()
def monitor(self):
self.logging.info("Connection monitor started.")
while self.loop():
sleep(5)
for conn in self.worker_instance.connection_list:
if not conn.connected:
self.logging.error("Connection to '%s' is dead. Trying to reconnect." % (conn.gearman_host))
try:
conn.connect()
self.logging.info("Connection to '%s' is restored." % (conn.gearman_host))
except Exception as err:
self.logging.error("Failed to reconnect to '%s'. Retry in 5 seconds. Reason: '%s'" % (conn.gearman_host, err))
else:
self.logging.debug("Connection to '%s' is alive." % (conn.gearman_host))
