def setUp(self): super(TestFunctional, self).setUp() if self.ssl: self.tmp_root = self.useFixture(fixtures.TempDir()).path root_subject, root_key = self.create_cert('root') self.create_cert('server', root_subject, root_key) self.create_cert('client', root_subject, root_key) self.create_cert('worker', root_subject, root_key) self.server = gear.Server( 0, os.path.join(self.tmp_root, 'server.key'), os.path.join(self.tmp_root, 'server.crt'), os.path.join(self.tmp_root, 'root.crt')) self.client = gear.Client('client') self.worker = gear.Worker('worker') self.client.addServer('127.0.0.1', self.server.port, os.path.join(self.tmp_root, 'client.key'), os.path.join(self.tmp_root, 'client.crt'), os.path.join(self.tmp_root, 'root.crt')) self.worker.addServer('127.0.0.1', self.server.port, os.path.join(self.tmp_root, 'worker.key'), os.path.join(self.tmp_root, 'worker.crt'), os.path.join(self.tmp_root, 'root.crt')) else: self.server = gear.Server(0) self.client = gear.Client('client') self.worker = gear.Worker('worker') self.client.addServer('127.0.0.1', self.server.port) self.worker.addServer('127.0.0.1', self.server.port) self.client.waitForServer() self.worker.waitForServer()
def run(self): self.log.debug("Node worker %s starting" % (self.name,)) server = self.config.get('gearman', 'server') if self.config.has_option('gearman', 'port'): port = self.config.get('gearman', 'port') else: port = 4730 self.worker = gear.Worker(self.name) self.worker.addServer(server, port) self.log.debug("Waiting for server") self.worker.waitForServer() self.log.debug("Registering") self.register() self.gearman_thread = threading.Thread(target=self.runGearman) self.gearman_thread.daemon = True self.gearman_thread.start() self.log.debug("Started") while self._running or not self.queue.empty(): try: self._runQueue() except Exception: self.log.exception("Exception in queue manager:")
def setup_gearman(self): hostname = os.uname()[1] self.gearman_worker = gear.Worker('turbo-hipster-manager-%s' % hostname) self.gearman_worker.addServer( self.worker_server.config['zuul_server']['gearman_host'], self.worker_server.config['zuul_server']['gearman_port'])
def setup_retriever(self): hostname = socket.gethostname() gearman_worker = gear.Worker(hostname + b'-pusher') gearman_worker.addServer(self.gearman_host, self.gearman_port) gearman_worker.registerFunction(b'push-log') self.retriever = LogRetriever(gearman_worker, self.filter_factories, self.logqueue)
def start(self): self._gearman_running = True self._zmq_running = True self._reaper_running = True self._command_running = True # Setup ZMQ self.zcontext = zmq.Context() self.zsocket = self.zcontext.socket(zmq.PUB) self.zsocket.bind("tcp://*:8888") # Setup Gearman server = self.config.get('gearman', 'server') if self.config.has_option('gearman', 'port'): port = self.config.get('gearman', 'port') else: port = 4730 self.worker = gear.Worker('Zuul Launch Server') self.worker.addServer(server, port) self.log.debug("Waiting for server") self.worker.waitForServer() self.log.debug("Registering") self.register() # Start command socket self.log.debug("Starting command processor") self.command_socket.start() self.command_thread = threading.Thread(target=self.runCommand) self.command_thread.daemon = True self.command_thread.start() # Load JJB config self.loadJobs() # Start ZMQ worker thread self.log.debug("Starting ZMQ processor") self.zmq_thread = threading.Thread(target=self.runZMQ) self.zmq_thread.daemon = True self.zmq_thread.start() # Start node worker reaper thread self.log.debug("Starting reaper") self.reaper_thread = threading.Thread(target=self.runReaper) self.reaper_thread.daemon = True self.reaper_thread.start() # Start Gearman worker thread self.log.debug("Starting worker") self.gearman_thread = threading.Thread(target=self.run) self.gearman_thread.daemon = True self.gearman_thread.start() # Start static workers for node in self.static_nodes.values(): self.log.debug("Creating static node with arguments: %s" % (node,)) self._launchWorker(node)
def setup_retriever(self): hostname = socket.gethostname() gearman_worker = gear.Worker(hostname + b'-pusher') gearman_worker.addServer(self.gearman_host, self.gearman_port) gearman_worker.registerFunction(b'push-subunit') subunit2sql_conf = self.config['config'] self.retriever = SubunitRetriever(gearman_worker, self.filter_factories, subunit2sql_conf, mqtt=self.mqtt)
def start(self): self._running = True server = self.config.get('gearman', 'server') if self.config.has_option('gearman', 'port'): port = self.config.get('gearman', 'port') else: port = 4730 self.worker = gear.Worker('Zuul RPC Listener') self.worker.addServer(server, port) self.worker.waitForServer() self.register() self.thread = threading.Thread(target=self.run) self.thread.daemon = True self.thread.start()
def worker(client_id, func_name, df, key, subset): worker = gear.Worker(client_id) worker.addServer(host=ct.GEARMAND_HOST, port=ct.GEARMAND_PORT) worker.registerFunction(func_name) redis = create_redis_obj() while True: job = worker.getJob() info = json.loads(job.arguments.decode('utf-8')) tmp_df = pandas.DataFrame(info, index=[0]) tmp_redis = redis.get(key) if tmp_redis is not None: df = _pickle.loads(tmp_redis) df = df.append(tmp_df) df = df.drop_duplicates(subset) redis.set(key, _pickle.dumps(df, 2)) job.sendWorkComplete()
def start(self): self._running = True server = self.config.get('gearman', 'server') if self.config.has_option('gearman', 'port'): port = self.config.get('gearman', 'port') else: port = 4730 self.worker = gear.Worker('Zuul Merger') self.worker.addServer(server, port) self.log.debug("Waiting for server") self.worker.waitForServer() self.log.debug("Registering") self.register() self.log.debug("Starting worker") self.thread = threading.Thread(target=self.run) self.thread.daemon = True self.thread.start()
def test_worker(self): tdir = self.useFixture(fixtures.TempDir()) dsn = 'sqlite:///%s/test.db' % tdir.path w = worker.GearhornWorker(client_id='test_worker', dsn=dsn) w._store.initialize_schema() self.addCleanup(w.shutdown) w.addServer('localhost', self.server.port) w.registerSubscriberFunctions() w.registerFanoutFunction() subw = gear.Worker('test_worker_subw') subw.addServer('localhost', self.server.port) subw.waitForServer() subw.registerFunction('broadcasts_test_receiver') subscribe_message = { 'client_id': 'test_receiver', 'topic': 'broadcasts' } subscribe_job = gear.Job( w.subscribe_name, arguments=json.dumps(subscribe_message).encode('utf-8')) self.client.submitJob(subscribe_job) # w should have this message only w.work() while not subscribe_job.complete: time.sleep(0.1) fanout_message = {'topic': 'broadcasts', 'payload': 'in payload'} job = gear.Job(w.fanout_name, json.dumps(fanout_message).encode('utf-8')) self.client.submitJob(job) # Thread in the background to wait for subw to complete job t = threading.Thread(target=w.work) t.start() broadcasted = subw.getJob() self.assertEqual('in payload', broadcasted.arguments) broadcasted.sendWorkComplete() # wait for complete to wind through tubes while not job.complete: time.sleep(0.1) self.assertFalse(job.failure) self.assertIsNone(job.exception) self.assertEqual([b'1'], job.data) t.join()
def test_subscriber(self): w = gear.Worker('test_subscriber_worker') w.registerFunction(worker.GearhornWorker.subscribe_name) self.addCleanup(w.shutdown) w.addServer('localhost', self.server.port) s = subscriber.GearhornSubscriber('test_subscriber') s.addServer('localhost', self.server.port) s.waitForServer() s.subscribe('a_topic') def _assertJob(): p = json.loads(j.arguments) self.assertIsInstance(p, dict) self.assertIn('topic', p) self.assertIn('client_id', p) self.assertEqual('a_topic', p['topic']) self.assertEqual('test_subscriber', p['client_id']) j = w.getJob() _assertJob() w.registerFunction(worker.GearhornWorker.unsubscribe_name) s.unsubscribe('a_topic') j = w.getJob() _assertJob()
def __init__(self, app): self.worker = gear.Worker('run_remote_test') self.worker.addServer(app.gearman_server) self.worker.registerFunction('run_remote_test')
import gear import json import sys sys.path.append('../api') from controllerBase import Controller worker = gear.Worker('apiWorker') worker.addServer(host='localhost', port=4730) worker.registerFunction('api') while True: job = worker.getJob() data = json.loads(job.arguments.decode('utf-8')) controller = Controller(data) data = controller.action() data = json.dumps(data) job.sendWorkComplete(bytes(data, 'utf8'))
def __init__(self, client_id): self.client_id = client_id self.worker = gear.Worker(client_id=client_id) client_client_id = '%s_subscriber' % (client_id) self.client = gear.Client(client_id=client_client_id)
import gear def task_listener_reverse(gearman_worker, gearman_job): print 'Reversing string: ' + gearman_job.data return gearman_job.data[::-1] worker = gear.Worker('reverser') worker.addServer('192.168.122.89') worker.registerFunction('reverse') while True: job = worker.getJob() job.sendWorkComplete(job.arguments[::-1])
def __init__(self, resoursesPath): self._clf = Classifier(resoursesPath) self._funcs = {} self._worker = gear.Worker('huaban-brain')
def setup_gearman(self): self.log.debug("Set up gearman worker") self.gearman_worker = gear.Worker(self.worker_server.worker_name) self.gearman_worker.addServer( self.worker_server.config['zuul_server']['gearman_host'], self.worker_server.config['zuul_server']['gearman_port'])