def _init(self): while True: while not self.is_available(): logging.error( "Gearman not available right now. Demon will sleep during {n} seconds" .format(n=settings.GEARMAN_RECONNECT_TIMEOUT)) gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT) logging.debug("Gearman worker try to connect {hosts}".format( hosts=', '.join(settings.GEARMAN['hosts']))) try: gm_worker = GearmanWorker(settings.GEARMAN['hosts']) gm_worker.set_client_id("socket_io_gearman_" + str(OPTIONS.port)) gm_worker.register_task("socket_io", GearmanListener.callback) logging.debug("Gearman worker was successfull created") return gm_worker except Exception, e: logging.error( "Error while initiation gearman worker connect with message: {message}" .format(message=e.message)) logging.debug("Demon will be sleep during {n} seconds".format( n=settings.GEARMAN_RECONNECT_TIMEOUT)) gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT)
myj = Jenkins(jenkins_data['url']) job = myj.get_job(jenkins_data['job_id']) #job.invoke(securitytoken=token, block=block) job.invoke(invoke_pre_check_delay=0) except: rev = "Not Happy!!!" return rev # Establish a connection with the job server on localhost--like the client, # multiple job servers can be used. worker = GearmanWorker(['localhost']) # register_task will tell the job server that this worker handles the "echo" # task worker.set_client_id('your_worker_client_id_name') worker.register_task('echo', task_listener_echo) worker.register_task('build:pep8', task_listener_build) worker.register_task('stop:jenkins_master.hp.com', task_listener_stop) worker.register_task('bravo', task_listener_echo) worker.register_task('reverse', task_listener_reverse) worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job) # Once setup is complete, begin working by consuming any tasks available # from the job server print 'working...' worker.work() # The worker will continue to run (waiting for new work) until exited by # code or an external signal is caught
#TODO: Log the error return str(e) ################################################ def task_listener_email(gearman_worker, gearman_job): #TODO: Log the call and params cleaned_words, reply, meta, infile, sample_text = process_words( gearman_job.data) #TODO: Log the call and params res = inverted_index(cleaned_words, reply, meta, infile, sample_text) print res #TODO: Log the result if res != 'Ok': # Log the stack trace and exception return 'NotOk' else: return res if __name__ == "__main__": # TODO: Daemonize # TODO: Ensure there are not multiple monitors for the same location (check for lock file) # TODO: Make the strings config driven # TODO: Have a meaningful try-except block gm_worker.set_client_id('inv-index-worker') gm_worker.register_task('invindex', task_listener_email) gm_worker.work()
def task_backtest(gearman_worker, gearman_job): symbol = ['000001', '603993'] bars = bindata.BackTestData(bindata.raw) # Apply our current strategy on the chosen stock pool rfs = CurrentStrategy(symbol, bars) # specify constraints, here is the default one cons = Constraint() # specify a naive optimizer opt = NaiveOptimizer(cons) data = json.loads(gearman_job.data) function_list = {} signal_generator = compile(data["code"], '', 'exec') exec signal_generator in function_list # Create a portfolio portfolio = MarketOnClosePortfolio(symbol, bars, rfs, \ opt, initial_capital=1000000.0) portfolio.strategy.sig_generator = function_list["generate_signals"] # Backtest our portfolio and store result in book book = portfolio.backtest_portfolio(worker=gearman_worker, job=gearman_job) ret = book.nav_to_json() return json.dumps(ret) if __name__ == "__main__": # gm_worker.set_client_id is optional gm_worker.set_client_id('python-worker') gm_worker.register_task('backtest', task_backtest) # Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity gm_worker.work()
topN_list = [{item['word_doc_hash'].split('_')[1]:item} for item in topN] # NOTE: Persist topN into redis redis_server.set(current_topn_pattern, str(topN_list)) # NOTE: remove the tbp_<word> from the to be processed list redis_server.delete(word) return 'Ok' ######################################################################## def task_listener_topncompute(gearman_worker, gearman_job): #TODO: Log the call and params res = topncompute(gearman_job.data) print res #TODO: Log the result if res != 'Ok': # Log the stack trace and exception return 'NotOk' else: return res if __name__ == "__main__": # TODO: Daemonize # TODO: Ensure there are not multiple monitors for the same location (check for lock file) # TODO: Make the strings config driven # TODO: Have a meaningful try-except block gm_worker.set_client_id('topn-worker') gm_worker.register_task('topncompute', task_listener_topncompute) gm_worker.work()
myj = Jenkins(jenkins_data['url']) job = myj.get_job(jenkins_data['job_id']) #job.invoke(securitytoken=token, block=block) job.invoke(invoke_pre_check_delay=0) except: rev="Not Happy!!!" return rev # Establish a connection with the job server on localhost--like the client, # multiple job servers can be used. worker = GearmanWorker(['localhost']) # register_task will tell the job server that this worker handles the "echo" # task worker.set_client_id('your_worker_client_id_name') worker.register_task('echo', task_listener_echo) worker.register_task('build:pep8', task_listener_build) worker.register_task('stop:jenkins_master.hp.com', task_listener_stop) worker.register_task('bravo', task_listener_echo) worker.register_task('reverse', task_listener_reverse) worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job) # Once setup is complete, begin working by consuming any tasks available # from the job server print 'working...' worker.work() # The worker will continue to run (waiting for new work) until exited by # code or an external signal is caught
return self.on_job_exception(current_job, sys.exc_info()) self.task_count +=1 if self.task_count >= 50: print "max tasks reached. exiting" sys.exit() return self.on_job_complete(current_job, job_result) GearmanWorker.task_count = 0 GearmanWorker.on_job_exception = on_job_exception GearmanWorker.on_job_execute = on_job_execute worker = GearmanWorker(JOBSERVER_LIST) worker.set_client_id("working_on_the_djangoroad") worker.register_task("add", add) worker.register_task("pre_schedule", pre_schedule) worker.register_task("row_calculator", row_calculator) worker.register_task("row_aggregator", row_aggregator) worker.register_task("echo", echo) print "working" #print dir(worker) #print worker.worker_abilities worker.work()