def setUp(self):
super(TestFunctional, self).setUp()
if self.ssl:
self.tmp_root = self.useFixture(fixtures.TempDir()).path
root_subject, root_key = self.create_cert('root')
self.create_cert('server', root_subject, root_key)
self.create_cert('client', root_subject, root_key)
self.create_cert('worker', root_subject, root_key)
self.server = gear.Server(
0,
os.path.join(self.tmp_root, 'server.key'),
os.path.join(self.tmp_root, 'server.crt'),
os.path.join(self.tmp_root, 'root.crt'))
self.client = gear.Client('client')
self.worker = gear.Worker('worker')
self.client.addServer('127.0.0.1', self.server.port,
os.path.join(self.tmp_root, 'client.key'),
os.path.join(self.tmp_root, 'client.crt'),
os.path.join(self.tmp_root, 'root.crt'))
self.worker.addServer('127.0.0.1', self.server.port,
os.path.join(self.tmp_root, 'worker.key'),
os.path.join(self.tmp_root, 'worker.crt'),
os.path.join(self.tmp_root, 'root.crt'))
else:
self.server = gear.Server(0)
self.client = gear.Client('client')
self.worker = gear.Worker('worker')
self.client.addServer('127.0.0.1', self.server.port)
self.worker.addServer('127.0.0.1', self.server.port)
self.client.waitForServer()
self.worker.waitForServer()
def run(self):
self.log.debug("Node worker %s starting" % (self.name,))
server = self.config.get('gearman', 'server')
if self.config.has_option('gearman', 'port'):
port = self.config.get('gearman', 'port')
else:
port = 4730
self.worker = gear.Worker(self.name)
self.worker.addServer(server, port)
self.log.debug("Waiting for server")
self.worker.waitForServer()
self.log.debug("Registering")
self.register()
self.gearman_thread = threading.Thread(target=self.runGearman)
self.gearman_thread.daemon = True
self.gearman_thread.start()
self.log.debug("Started")
while self._running or not self.queue.empty():
try:
self._runQueue()
except Exception:
self.log.exception("Exception in queue manager:")
def setup_gearman(self):
hostname = os.uname()[1]
self.gearman_worker = gear.Worker('turbo-hipster-manager-%s' %
hostname)
self.gearman_worker.addServer(
self.worker_server.config['zuul_server']['gearman_host'],
self.worker_server.config['zuul_server']['gearman_port'])
def setup_retriever(self):
hostname = socket.gethostname()
gearman_worker = gear.Worker(hostname + b'-pusher')
gearman_worker.addServer(self.gearman_host, self.gearman_port)
gearman_worker.registerFunction(b'push-log')
self.retriever = LogRetriever(gearman_worker, self.filter_factories,
self.logqueue)
def start(self):
self._gearman_running = True
self._zmq_running = True
self._reaper_running = True
self._command_running = True
# Setup ZMQ
self.zcontext = zmq.Context()
self.zsocket = self.zcontext.socket(zmq.PUB)
self.zsocket.bind("tcp://*:8888")
# Setup Gearman
server = self.config.get('gearman', 'server')
if self.config.has_option('gearman', 'port'):
port = self.config.get('gearman', 'port')
else:
port = 4730
self.worker = gear.Worker('Zuul Launch Server')
self.worker.addServer(server, port)
self.log.debug("Waiting for server")
self.worker.waitForServer()
self.log.debug("Registering")
self.register()
# Start command socket
self.log.debug("Starting command processor")
self.command_socket.start()
self.command_thread = threading.Thread(target=self.runCommand)
self.command_thread.daemon = True
self.command_thread.start()
# Load JJB config
self.loadJobs()
# Start ZMQ worker thread
self.log.debug("Starting ZMQ processor")
self.zmq_thread = threading.Thread(target=self.runZMQ)
self.zmq_thread.daemon = True
self.zmq_thread.start()
# Start node worker reaper thread
self.log.debug("Starting reaper")
self.reaper_thread = threading.Thread(target=self.runReaper)
self.reaper_thread.daemon = True
self.reaper_thread.start()
# Start Gearman worker thread
self.log.debug("Starting worker")
self.gearman_thread = threading.Thread(target=self.run)
self.gearman_thread.daemon = True
self.gearman_thread.start()
# Start static workers
for node in self.static_nodes.values():
self.log.debug("Creating static node with arguments: %s" % (node,))
self._launchWorker(node)
def setup_retriever(self):
hostname = socket.gethostname()
gearman_worker = gear.Worker(hostname + b'-pusher')
gearman_worker.addServer(self.gearman_host, self.gearman_port)
gearman_worker.registerFunction(b'push-subunit')
subunit2sql_conf = self.config['config']
self.retriever = SubunitRetriever(gearman_worker,
self.filter_factories,
subunit2sql_conf,
mqtt=self.mqtt)
def start(self):
self._running = True
server = self.config.get('gearman', 'server')
if self.config.has_option('gearman', 'port'):
port = self.config.get('gearman', 'port')
else:
port = 4730
self.worker = gear.Worker('Zuul RPC Listener')
self.worker.addServer(server, port)
self.worker.waitForServer()
self.register()
self.thread = threading.Thread(target=self.run)
self.thread.daemon = True
self.thread.start()
def worker(client_id, func_name, df, key, subset):
worker = gear.Worker(client_id)
worker.addServer(host=ct.GEARMAND_HOST, port=ct.GEARMAND_PORT)
worker.registerFunction(func_name)
redis = create_redis_obj()
while True:
job = worker.getJob()
info = json.loads(job.arguments.decode('utf-8'))
tmp_df = pandas.DataFrame(info, index=[0])
tmp_redis = redis.get(key)
if tmp_redis is not None: df = _pickle.loads(tmp_redis)
df = df.append(tmp_df)
df = df.drop_duplicates(subset)
redis.set(key, _pickle.dumps(df, 2))
job.sendWorkComplete()
def start(self):
self._running = True
server = self.config.get('gearman', 'server')
if self.config.has_option('gearman', 'port'):
port = self.config.get('gearman', 'port')
else:
port = 4730
self.worker = gear.Worker('Zuul Merger')
self.worker.addServer(server, port)
self.log.debug("Waiting for server")
self.worker.waitForServer()
self.log.debug("Registering")
self.register()
self.log.debug("Starting worker")
self.thread = threading.Thread(target=self.run)
self.thread.daemon = True
self.thread.start()
def test_worker(self):
tdir = self.useFixture(fixtures.TempDir())
dsn = 'sqlite:///%s/test.db' % tdir.path
w = worker.GearhornWorker(client_id='test_worker', dsn=dsn)
w._store.initialize_schema()
self.addCleanup(w.shutdown)
w.addServer('localhost', self.server.port)
w.registerSubscriberFunctions()
w.registerFanoutFunction()
subw = gear.Worker('test_worker_subw')
subw.addServer('localhost', self.server.port)
subw.waitForServer()
subw.registerFunction('broadcasts_test_receiver')
subscribe_message = {
'client_id': 'test_receiver',
'topic': 'broadcasts'
}
subscribe_job = gear.Job(
w.subscribe_name,
arguments=json.dumps(subscribe_message).encode('utf-8'))
self.client.submitJob(subscribe_job)
# w should have this message only
w.work()
while not subscribe_job.complete:
time.sleep(0.1)
fanout_message = {'topic': 'broadcasts', 'payload': 'in payload'}
job = gear.Job(w.fanout_name,
json.dumps(fanout_message).encode('utf-8'))
self.client.submitJob(job)
# Thread in the background to wait for subw to complete job
t = threading.Thread(target=w.work)
t.start()
broadcasted = subw.getJob()
self.assertEqual('in payload', broadcasted.arguments)
broadcasted.sendWorkComplete()
# wait for complete to wind through tubes
while not job.complete:
time.sleep(0.1)
self.assertFalse(job.failure)
self.assertIsNone(job.exception)
self.assertEqual([b'1'], job.data)
t.join()
def test_subscriber(self):
w = gear.Worker('test_subscriber_worker')
w.registerFunction(worker.GearhornWorker.subscribe_name)
self.addCleanup(w.shutdown)
w.addServer('localhost', self.server.port)
s = subscriber.GearhornSubscriber('test_subscriber')
s.addServer('localhost', self.server.port)
s.waitForServer()
s.subscribe('a_topic')
def _assertJob():
p = json.loads(j.arguments)
self.assertIsInstance(p, dict)
self.assertIn('topic', p)
self.assertIn('client_id', p)
self.assertEqual('a_topic', p['topic'])
self.assertEqual('test_subscriber', p['client_id'])
j = w.getJob()
_assertJob()
w.registerFunction(worker.GearhornWorker.unsubscribe_name)
s.unsubscribe('a_topic')
j = w.getJob()
_assertJob()
def __init__(self, app):
self.worker = gear.Worker('run_remote_test')
self.worker.addServer(app.gearman_server)
self.worker.registerFunction('run_remote_test')
import gear
import json
import sys
sys.path.append('../api')
from controllerBase import Controller
worker = gear.Worker('apiWorker')
worker.addServer(host='localhost', port=4730)
worker.registerFunction('api')
while True:
job = worker.getJob()
data = json.loads(job.arguments.decode('utf-8'))
controller = Controller(data)
data = controller.action()
data = json.dumps(data)
job.sendWorkComplete(bytes(data, 'utf8'))
def __init__(self, client_id):
self.client_id = client_id
self.worker = gear.Worker(client_id=client_id)
client_client_id = '%s_subscriber' % (client_id)
self.client = gear.Client(client_id=client_client_id)
import gear
def task_listener_reverse(gearman_worker, gearman_job):
print 'Reversing string: ' + gearman_job.data
return gearman_job.data[::-1]
worker = gear.Worker('reverser')
worker.addServer('192.168.122.89')
worker.registerFunction('reverse')
while True:
job = worker.getJob()
job.sendWorkComplete(job.arguments[::-1])
def __init__(self, resoursesPath):
self._clf = Classifier(resoursesPath)
self._funcs = {}
self._worker = gear.Worker('huaban-brain')
def setup_gearman(self):
self.log.debug("Set up gearman worker")
self.gearman_worker = gear.Worker(self.worker_server.worker_name)
self.gearman_worker.addServer(
self.worker_server.config['zuul_server']['gearman_host'],
self.worker_server.config['zuul_server']['gearman_port'])
