class GEWorker(daemon.Daemon): "Our worker daemon - takes Gearman jobs and does them" def __init__(self): super(GEWorker, self).__init__() self.worker = GearmanWorker(settings.GEARMAN_SERVERS) self.worker.register_function(messaging.SUBJECT_THREAD_COUNT, self.thread_count) self.worker.register_function(messaging.SUBJECT_THREAD_NAMES, self.thread_names) self.worker.register_function(messaging.SUBJECT_AVG, self.average) self._set_transaction_isolation() def _set_transaction_isolation(self): """ Defaut transaction isolation on MySQL InnoDB is REPEATABLE-READ, which means a connection always gets the same result for a given query, even if the data has changed. This daemon runs for a long time, we need to see db changes, so change to READ-COMMITTED. """ cur = connection.cursor() cur.execute("set session transaction isolation level read committed") cur.close() def run(self): """Entry point for daemon.Daemon subclasses - main method""" self.worker.work() # Never returns def wait_for_thread(self): 'Sleeps until a thread is available. Gearman will queue the requests whilst we pause here' if settings.DEBUG: connection.queries = [] # Prevent query list growing indefinitely running_threads = threading.active_count() while running_threads >= settings.MAX_WORKER_THREADS: logging.debug( 'Waiting for thread. Currently %s: %s' % (running_threads, self.thread_names()) ) time.sleep(1) def thread_count(self, job): 'Returns number of threads currently running' return str(threading.active_count()) +'\n' def thread_names(self, job): 'Returns an array of active thread names' return str( ', '.join([thread.name for thread in threading.enumerate()]) ) +'\n' def average(self, job): """ Calculates daily indicator average for group and overall @job.arg Id of the new Answer to include. """ self.wait_for_thread() worker_thread = AverageWorker(job.arg) worker_thread.start()
class TestGearman(GearmanTestCase): def setUp(self): self.start_server() self.last_exception = (None, None) self.worker = GearmanWorker(job_servers) self.worker.register_function("echo", echo) self.worker.register_function("fail", fail) self.worker.register_function("sleep", sleep, timeout=1) self.worker.register_class(ObjectWorker()) self.worker.register_class(ClassWorker()) class Hooks(object): @staticmethod def start(job): pass @staticmethod def complete(job, res): pass @staticmethod def fail(job, exc): self.last_exception = (job.func, exc) import thread self.worker_thread = thread.start_new_thread(self.worker.work, tuple(), dict(hooks=Hooks)) # TODO: Shouldn't use threads.. but we do for now (also, the thread is never terminated) self.client = GearmanClient(job_servers) def tearDown(self): del self.worker del self.client self.stop_server() def testComplete(self): self.failUnlessEqual(self.client.do_task(Task("echo", "bar")), 'bar') def testFail(self): self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("fail", "bar"))) # self.failUnlessEqual(self.last_exception[0], "fail") def testCompleteAfterFail(self): self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("fail", "bar"))) self.failUnlessEqual(self.client.do_task(Task("echo", "bar")), 'bar') def testTimeout(self): self.failUnlessEqual(self.client.do_task(Task("sleep", "0.1")), '0.1') self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("sleep", "1.5"))) def testCall(self): self.failUnlessEqual(self.client("echo", "bar"), 'bar') def testObjectWorker(self): self.failUnlessEqual(self.client("ObjectWorker.echo", "foo"), "foo") def testClassWorker(self): self.failUnlessEqual(self.client("ClassWorker.echo", "foo"), "foo")
def main(): print("Running Worker .....") print("options.logging = %s" % options.logging) from demisaucepy import cache_setup cache_setup.load_cache() #global app #app = AppBase() logging.info("site_root = %s" % options.site_root) logging.info("smtp servers = %s" % options.smtp_server) logging.info("cache servers = %s" % options.memcached_servers) logging.info("gearman servers 2 = %s" % (options.gearman_servers)) logging.error("where does this go in supervisord?") worker = GearmanWorker(options.gearman_servers) worker.register_function("email_send", emailer.email_send) worker.register_function("image_resize", assets.image_resize) worker.register_function("echo", echo) from demisauce.model import actions actions.register_workers(worker) worker.work()
def create_worker(): discover_workers() worker = GearmanWorker(settings.GEARMAN_SERVERS) for id, func in workers.iteritems(): worker.register_function(id, func) return worker
def create_worker(): discover_workers() worker = GearmanWorker(settings.GEARMAN_SERVERS) for id, func in workers.iteritems(): worker.register_function(id, func) return worker
def handle(self, *args, **options): print "worker started" worker = GearmanWorker(["127.0.0.1"]) worker.register_function("download", download) worker.work()
def preprocess_gm(job): try: job.status(0, 0) myjob = GmJob(job) # Gearman job myjob.run() result = myjob.finish() except Exception, e: print e print 'worker finished job:%s' % job.handle print '-' * 80 return result def preprocess_sa(): myjob = SaJob() # 单机job myjob.run() result = myjob.finish() wf = open('tmp.txt', 'w') wf.write(result) return result if __name__ == '__main__': if len(sys.argv) > 1: preprocess_sa() else: worker = GearmanWorker(['10.61.0.145']) print "worker started." worker.register_function('crawl', preprocess_gm) worker.work()
# work function for gearman def preprocess_gm( job ): try: job.status(0,0) myjob = GmJob( job ) # Gearman job myjob.run() result = myjob.finish() except Exception,e: print e print 'worker finished job:%s' % job.handle print '-'*80 return result def preprocess_sa(): myjob = SaJob() # 单机job myjob.run() result = myjob.finish() wf = open('tmp.txt','w') wf.write(result) return result if __name__ == '__main__': if len(sys.argv)>1: preprocess_sa() else: worker = GearmanWorker( ['10.61.0.145'] ) print "worker started." worker.register_function( 'crawl',preprocess_gm ) worker.work()
def handle(self, *args, **options): print "worker started" worker = GearmanWorker(["127.0.0.1"]) worker.register_function("download", download) worker.work()
from gearman import GearmanWorker def speak(job): r = 'Hello %s' % job.arg print r return r worker = GearmanWorker(["127.0.0.1"]) worker.register_function('speak', speak, timeout=3) worker.work()