def worker_killer_task(config, master_address, worker_address):
from scannerpy import ProtobufGenerator, Config, start_worker
import time
import grpc
import subprocess
import signal
import os
c = Config(None)
import scanner.metadata_pb2 as metadata_types
import scanner.engine.rpc_pb2 as rpc_types
import scanner.types_pb2 as misc_types
import libscanner as bindings
protobufs = ProtobufGenerator(config)
# Kill worker
channel = grpc.insecure_channel(
worker_address,
options=[('grpc.max_message_length', 24499183 * 2)])
worker = protobufs.WorkerStub(channel)
try:
worker.Shutdown(protobufs.Empty())
except grpc.RpcError as e:
status = e.code()
if status == grpc.StatusCode.UNAVAILABLE:
print('could not shutdown worker!')
exit(1)
else:
raise ScannerException('Worker errored with status: {}'
.format(status))
# Spawn a worker that we will force kill
script_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.devnull, 'w') as fp:
p = subprocess.Popen(
['python ' + script_dir + '/spawn_worker.py'],
shell=True,
stdout=fp, stderr=fp,
preexec_fn=os.setsid)
# Wait a bit for the worker to do its thing
time.sleep(10)
# Force kill worker process to trigger fault tolerance
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
p.communicate()
# Wait for fault tolerance to kick in
time.sleep(25)
# Spawn the worker again
subprocess.call(['python ' + script_dir + '/spawn_worker.py'],
shell=True)
def worker_shutdown_task(config, master_address, worker_address):
from scannerpy import ProtobufGenerator, Config, start_worker
import time
import grpc
import subprocess
c = Config(None)
import scanner.metadata_pb2 as metadata_types
import scanner.engine.rpc_pb2 as rpc_types
import scanner.types_pb2 as misc_types
import libscanner as bindings
protobufs = ProtobufGenerator(config)
# Wait to kill worker
time.sleep(8)
# Kill worker
channel = grpc.insecure_channel(
worker_address,
options=[('grpc.max_message_length', 24499183 * 2)])
worker = protobufs.WorkerStub(channel)
try:
worker.Shutdown(protobufs.Empty())
except grpc.RpcError as e:
status = e.code()
if status == grpc.StatusCode.UNAVAILABLE:
print('could not shutdown worker!')
exit(1)
else:
raise ScannerException('Worker errored with status: {}'
.format(status))
# Wait a bit
time.sleep(15)
script_dir = os.path.dirname(os.path.realpath(__file__))
subprocess.call(['python ' + script_dir + '/spawn_worker.py'],
shell=True)