class GEWorker(daemon.Daemon):
"Our worker daemon - takes Gearman jobs and does them"
def __init__(self):
super(GEWorker, self).__init__()
self.worker = GearmanWorker(settings.GEARMAN_SERVERS)
self.worker.register_function(messaging.SUBJECT_THREAD_COUNT, self.thread_count)
self.worker.register_function(messaging.SUBJECT_THREAD_NAMES, self.thread_names)
self.worker.register_function(messaging.SUBJECT_AVG, self.average)
self._set_transaction_isolation()
def _set_transaction_isolation(self):
"""
Defaut transaction isolation on MySQL InnoDB is REPEATABLE-READ, which means
a connection always gets the same result for a given query, even if the data has
changed.
This daemon runs for a long time, we need to see db changes, so change to READ-COMMITTED.
"""
cur = connection.cursor()
cur.execute("set session transaction isolation level read committed")
cur.close()
def run(self):
"""Entry point for daemon.Daemon subclasses - main method"""
self.worker.work() # Never returns
def wait_for_thread(self):
'Sleeps until a thread is available. Gearman will queue the requests whilst we pause here'
if settings.DEBUG:
connection.queries = [] # Prevent query list growing indefinitely
running_threads = threading.active_count()
while running_threads >= settings.MAX_WORKER_THREADS:
logging.debug(
'Waiting for thread. Currently %s: %s' %
(running_threads, self.thread_names()) )
time.sleep(1)
def thread_count(self, job):
'Returns number of threads currently running'
return str(threading.active_count()) +'\n'
def thread_names(self, job):
'Returns an array of active thread names'
return str( ', '.join([thread.name for thread in threading.enumerate()]) ) +'\n'
def average(self, job):
""" Calculates daily indicator average for group and overall
@job.arg Id of the new Answer to include.
"""
self.wait_for_thread()
worker_thread = AverageWorker(job.arg)
worker_thread.start()
class TestGearman(GearmanTestCase):
def setUp(self):
self.start_server()
self.last_exception = (None, None)
self.worker = GearmanWorker(job_servers)
self.worker.register_function("echo", echo)
self.worker.register_function("fail", fail)
self.worker.register_function("sleep", sleep, timeout=1)
self.worker.register_class(ObjectWorker())
self.worker.register_class(ClassWorker())
class Hooks(object):
@staticmethod
def start(job):
pass
@staticmethod
def complete(job, res):
pass
@staticmethod
def fail(job, exc):
self.last_exception = (job.func, exc)
import thread
self.worker_thread = thread.start_new_thread(self.worker.work, tuple(), dict(hooks=Hooks)) # TODO: Shouldn't use threads.. but we do for now (also, the thread is never terminated)
self.client = GearmanClient(job_servers)
def tearDown(self):
del self.worker
del self.client
self.stop_server()
def testComplete(self):
self.failUnlessEqual(self.client.do_task(Task("echo", "bar")), 'bar')
def testFail(self):
self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("fail", "bar")))
# self.failUnlessEqual(self.last_exception[0], "fail")
def testCompleteAfterFail(self):
self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("fail", "bar")))
self.failUnlessEqual(self.client.do_task(Task("echo", "bar")), 'bar')
def testTimeout(self):
self.failUnlessEqual(self.client.do_task(Task("sleep", "0.1")), '0.1')
self.failUnlessRaises(self.client.TaskFailed, lambda:self.client.do_task(Task("sleep", "1.5")))
def testCall(self):
self.failUnlessEqual(self.client("echo", "bar"), 'bar')
def testObjectWorker(self):
self.failUnlessEqual(self.client("ObjectWorker.echo", "foo"), "foo")
def testClassWorker(self):
self.failUnlessEqual(self.client("ClassWorker.echo", "foo"), "foo")
def main():
print("Running Worker .....")
print("options.logging = %s" % options.logging)
from demisaucepy import cache_setup
cache_setup.load_cache()
#global app
#app = AppBase()
logging.info("site_root = %s" % options.site_root)
logging.info("smtp servers = %s" % options.smtp_server)
logging.info("cache servers = %s" % options.memcached_servers)
logging.info("gearman servers 2 = %s" % (options.gearman_servers))
logging.error("where does this go in supervisord?")
worker = GearmanWorker(options.gearman_servers)
worker.register_function("email_send", emailer.email_send)
worker.register_function("image_resize", assets.image_resize)
worker.register_function("echo", echo)
from demisauce.model import actions
actions.register_workers(worker)
worker.work()
def create_worker():
discover_workers()
worker = GearmanWorker(settings.GEARMAN_SERVERS)
for id, func in workers.iteritems():
worker.register_function(id, func)
return worker
def create_worker():
discover_workers()
worker = GearmanWorker(settings.GEARMAN_SERVERS)
for id, func in workers.iteritems():
worker.register_function(id, func)
return worker
def handle(self, *args, **options):
print "worker started"
worker = GearmanWorker(["127.0.0.1"])
worker.register_function("download", download)
worker.work()
def preprocess_gm(job):
try:
job.status(0, 0)
myjob = GmJob(job) # Gearman job
myjob.run()
result = myjob.finish()
except Exception, e:
print e
print 'worker finished job:%s' % job.handle
print '-' * 80
return result
def preprocess_sa():
myjob = SaJob() # 单机job
myjob.run()
result = myjob.finish()
wf = open('tmp.txt', 'w')
wf.write(result)
return result
if __name__ == '__main__':
if len(sys.argv) > 1:
preprocess_sa()
else:
worker = GearmanWorker(['10.61.0.145'])
print "worker started."
worker.register_function('crawl', preprocess_gm)
worker.work()
# work function for gearman
def preprocess_gm( job ):
try:
job.status(0,0)
myjob = GmJob( job ) # Gearman job
myjob.run()
result = myjob.finish()
except Exception,e:
print e
print 'worker finished job:%s' % job.handle
print '-'*80
return result
def preprocess_sa():
myjob = SaJob() # 单机job
myjob.run()
result = myjob.finish()
wf = open('tmp.txt','w')
wf.write(result)
return result
if __name__ == '__main__':
if len(sys.argv)>1:
preprocess_sa()
else:
worker = GearmanWorker( ['10.61.0.145'] )
print "worker started."
worker.register_function( 'crawl',preprocess_gm )
worker.work()
def handle(self, *args, **options):
print "worker started"
worker = GearmanWorker(["127.0.0.1"])
worker.register_function("download", download)
worker.work()
from gearman import GearmanWorker
def speak(job):
r = 'Hello %s' % job.arg
print r
return r
worker = GearmanWorker(["127.0.0.1"])
worker.register_function('speak', speak, timeout=3)
worker.work()
