def test_get_worker_data(self):
worker = self.create_worker()
additional_data = {'test': 123}
worker_data = worker._get_worker_data(additional_data)
self.assertEqual(worker_data['main']['uuid'], worker.uuid)
self.assertAlmostEqual(worker_data['heart_beat_dt'], now(), delta=datetime.timedelta(seconds=1))
self.assertEqual(worker_data['test'], additional_data['test'])
def test_setup(self, register_worker_mock):
worker = BaseWorker()
worker.setup()
self.assertTrue(worker.id)
self.assertTrue(worker.uuid)
self.assertAlmostEqual(worker.created_dt, now(), delta=datetime.timedelta(seconds=1))
self.assertTrue(worker.host_name)
self.assertTrue(register_worker_mock.called)
def _init_worker(self, worker_id=None, verbose=False):
self.uuid = str(uuid.uuid4())
if worker_id is None:
self.id = self.uuid
else:
self.id = worker_id
self.created_dt = now()
self.host_name = socket.gethostname()
self.stop_event = multiprocessing.Event()
self.verbose = verbose
def test_get_worker_data(self):
worker = self.create_worker()
additional_data = {'test': 123}
worker_data = worker._get_worker_data(additional_data)
self.assertEqual(worker_data['main']['uuid'], worker.uuid)
self.assertAlmostEqual(worker_data['heart_beat_dt'],
now(),
delta=datetime.timedelta(seconds=1))
self.assertEqual(worker_data['test'], additional_data['test'])
def _register_start_task(self, task_data, task_type):
task_name = self._get_task_name(task_data, task_type)
if self.verbose:
print("Worker %s. Received request: %s for %s. %s" % (self.id, task_name, task_data["host_address"], now()))
self._emit_worker({"start_task": task_data})
task_data["start_dt"] = now()
task_data["worker"] = self._get_worker_data()["main"]
self._emit_task(task_data, task_type)
def test_setup(self, register_worker_mock):
worker = BaseWorker()
worker.setup()
self.assertTrue(worker.id)
self.assertTrue(worker.uuid)
self.assertAlmostEqual(worker.created_dt,
now(),
delta=datetime.timedelta(seconds=1))
self.assertTrue(worker.host_name)
self.assertTrue(register_worker_mock.called)
def test_heart_beat(self, sleep_mock, emit_mock, is_stopped_mock):
is_stopped_mock.side_effect = (False, True)
worker = BaseWorker()
worker._init_worker()
worker._heart_beat()
self.assertEqual(WORKER_EVENT, emit_mock.call_args[0][0])
worker_data = json.loads(emit_mock.call_args[0][1],
object_hook=json_util.object_hook)
self.assertAlmostEqual(now(),
worker_data['heart_beat_dt'],
delta=datetime.timedelta(seconds=1))
async def _receive_tasks(self):
task_socket = self._get_task_socket()
try:
loop = asyncio.get_event_loop()
while True:
data = await self.async_recv_pull_msg(task_socket)
task_data = data['task']
task_type = data['type']
self._register_start_task(task_data, task_type)
loop.create_task(self._process_task(task_data, task_type))
finally:
self._emit_worker({'stop_dt': now()})
task_socket.close()
def _get_worker_data(self, data=None):
worker_data = {
"main": {
"id": str(self.id),
"uuid": self.uuid,
"created_dt": self.created_dt,
"host_name": self.host_name,
"type": self.worker_type,
},
"heart_beat_dt": now(),
}
if data:
worker_data.update(data)
return worker_data
async def _receive_tasks(self):
task_socket = self._get_task_socket()
try:
loop = asyncio.get_event_loop()
while True:
data = await self.async_recv_pull_msg(task_socket)
task_data = data['task']
task_type = data['type']
self._register_start_task(task_data, task_type)
loop.create_task(self._process_task(task_data, task_type))
finally:
self._emit_worker({'stop_dt': now()})
task_socket.close()
def test_register_start_task(self, emit_mock):
worker = self.create_worker()
data = self.monit_task.get_data()
worker._register_start_task(data['task'], data['type'])
worker_data = self.get_emit_args(emit_mock, WORKER_EVENT)
self.assertIn('start_task', worker_data)
task_data = self.get_emit_args(emit_mock, TASK_EVENT)['task']
self.assertAlmostEqual(task_data['start_dt'],
now(),
delta=datetime.timedelta(seconds=1))
self.assertEqual(task_data['worker']['uuid'], worker.uuid)
def test_heart_beat(self, sleep_mock, emit_mock, is_stopped_mock):
is_stopped_mock.side_effect = (False, True)
worker = BaseWorker()
worker._init_worker()
worker._heart_beat()
self.assertEqual(WORKER_EVENT, emit_mock.call_args[0][0])
worker_data = json.loads(emit_mock.call_args[0][1], object_hook=json_util.object_hook)
self.assertAlmostEqual(
now(),
worker_data['heart_beat_dt'],
delta=datetime.timedelta(seconds=1)
)
def _receive_tasks(self):
task_socket = self._get_task_socket()
try:
while not self.is_stopped():
msg = task_socket.recv_string()
msg = msg.decode("utf-8")
data = json.loads(msg, object_hook=json_util.object_hook)
if self._is_stop_worker_msg(data):
return
# print('Worker %s receive task' % self.uuid, data)
task_data = data["task"]
task_type = data["type"]
self._register_start_task(task_data, task_type)
self._process_task(task_data, task_type)
finally:
self._emit_worker({"stop_dt": now()})
task_socket.close()
def test_register_start_task(self, emit_mock):
worker = self.create_worker()
data = self.monit_task.get_data()
worker._register_start_task(data['task'], data['type'])
worker_data = self.get_emit_args(emit_mock, WORKER_EVENT)
self.assertIn('start_task', worker_data)
task_data = self.get_emit_args(emit_mock, TASK_EVENT)['task']
self.assertAlmostEqual(
task_data['start_dt'],
now(),
delta=datetime.timedelta(seconds=1)
)
self.assertEqual(
task_data['worker']['uuid'],
worker.uuid
)
def __init__(self, level=None, dt=None, extra=None):
self.level = level or LEVEL_OK
self.extra = extra
self.dt = dt or now()