def _init(self):
while True:
while not self.is_available():
logging.error(
"Gearman not available right now. Demon will sleep during {n} seconds"
.format(n=settings.GEARMAN_RECONNECT_TIMEOUT))
gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT)
logging.debug("Gearman worker try to connect {hosts}".format(
hosts=', '.join(settings.GEARMAN['hosts'])))
try:
gm_worker = GearmanWorker(settings.GEARMAN['hosts'])
gm_worker.set_client_id("socket_io_gearman_" +
str(OPTIONS.port))
gm_worker.register_task("socket_io", GearmanListener.callback)
logging.debug("Gearman worker was successfull created")
return gm_worker
except Exception, e:
logging.error(
"Error while initiation gearman worker connect with message: {message}"
.format(message=e.message))
logging.debug("Demon will be sleep during {n} seconds".format(
n=settings.GEARMAN_RECONNECT_TIMEOUT))
gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT)
myj = Jenkins(jenkins_data['url'])
job = myj.get_job(jenkins_data['job_id'])
#job.invoke(securitytoken=token, block=block)
job.invoke(invoke_pre_check_delay=0)
except:
rev = "Not Happy!!!"
return rev
# Establish a connection with the job server on localhost--like the client,
# multiple job servers can be used.
worker = GearmanWorker(['localhost'])
# register_task will tell the job server that this worker handles the "echo"
# task
worker.set_client_id('your_worker_client_id_name')
worker.register_task('echo', task_listener_echo)
worker.register_task('build:pep8', task_listener_build)
worker.register_task('stop:jenkins_master.hp.com', task_listener_stop)
worker.register_task('bravo', task_listener_echo)
worker.register_task('reverse', task_listener_reverse)
worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job)
# Once setup is complete, begin working by consuming any tasks available
# from the job server
print 'working...'
worker.work()
# The worker will continue to run (waiting for new work) until exited by
# code or an external signal is caught
#TODO: Log the error
return str(e)
################################################
def task_listener_email(gearman_worker, gearman_job):
#TODO: Log the call and params
cleaned_words, reply, meta, infile, sample_text = process_words(
gearman_job.data)
#TODO: Log the call and params
res = inverted_index(cleaned_words, reply, meta, infile, sample_text)
print res
#TODO: Log the result
if res != 'Ok':
# Log the stack trace and exception
return 'NotOk'
else:
return res
if __name__ == "__main__":
# TODO: Daemonize
# TODO: Ensure there are not multiple monitors for the same location (check for lock file)
# TODO: Make the strings config driven
# TODO: Have a meaningful try-except block
gm_worker.set_client_id('inv-index-worker')
gm_worker.register_task('invindex', task_listener_email)
gm_worker.work()
def task_backtest(gearman_worker, gearman_job):
symbol = ['000001', '603993']
bars = bindata.BackTestData(bindata.raw)
# Apply our current strategy on the chosen stock pool
rfs = CurrentStrategy(symbol, bars)
# specify constraints, here is the default one
cons = Constraint()
# specify a naive optimizer
opt = NaiveOptimizer(cons)
data = json.loads(gearman_job.data)
function_list = {}
signal_generator = compile(data["code"], '', 'exec')
exec signal_generator in function_list
# Create a portfolio
portfolio = MarketOnClosePortfolio(symbol, bars, rfs, \
opt, initial_capital=1000000.0)
portfolio.strategy.sig_generator = function_list["generate_signals"]
# Backtest our portfolio and store result in book
book = portfolio.backtest_portfolio(worker=gearman_worker, job=gearman_job)
ret = book.nav_to_json()
return json.dumps(ret)
if __name__ == "__main__":
# gm_worker.set_client_id is optional
gm_worker.set_client_id('python-worker')
gm_worker.register_task('backtest', task_backtest)
# Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity
gm_worker.work()
topN_list = [{item['word_doc_hash'].split('_')[1]:item} for item in topN]
# NOTE: Persist topN into redis
redis_server.set(current_topn_pattern, str(topN_list))
# NOTE: remove the tbp_<word> from the to be processed list
redis_server.delete(word)
return 'Ok'
########################################################################
def task_listener_topncompute(gearman_worker, gearman_job):
#TODO: Log the call and params
res = topncompute(gearman_job.data)
print res
#TODO: Log the result
if res != 'Ok':
# Log the stack trace and exception
return 'NotOk'
else:
return res
if __name__ == "__main__":
# TODO: Daemonize
# TODO: Ensure there are not multiple monitors for the same location (check for lock file)
# TODO: Make the strings config driven
# TODO: Have a meaningful try-except block
gm_worker.set_client_id('topn-worker')
gm_worker.register_task('topncompute', task_listener_topncompute)
gm_worker.work()
myj = Jenkins(jenkins_data['url'])
job = myj.get_job(jenkins_data['job_id'])
#job.invoke(securitytoken=token, block=block)
job.invoke(invoke_pre_check_delay=0)
except:
rev="Not Happy!!!"
return rev
# Establish a connection with the job server on localhost--like the client,
# multiple job servers can be used.
worker = GearmanWorker(['localhost'])
# register_task will tell the job server that this worker handles the "echo"
# task
worker.set_client_id('your_worker_client_id_name')
worker.register_task('echo', task_listener_echo)
worker.register_task('build:pep8', task_listener_build)
worker.register_task('stop:jenkins_master.hp.com', task_listener_stop)
worker.register_task('bravo', task_listener_echo)
worker.register_task('reverse', task_listener_reverse)
worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job)
# Once setup is complete, begin working by consuming any tasks available
# from the job server
print 'working...'
worker.work()
# The worker will continue to run (waiting for new work) until exited by
# code or an external signal is caught
return self.on_job_exception(current_job, sys.exc_info())
self.task_count +=1
if self.task_count >= 50:
print "max tasks reached. exiting"
sys.exit()
return self.on_job_complete(current_job, job_result)
GearmanWorker.task_count = 0
GearmanWorker.on_job_exception = on_job_exception
GearmanWorker.on_job_execute = on_job_execute
worker = GearmanWorker(JOBSERVER_LIST)
worker.set_client_id("working_on_the_djangoroad")
worker.register_task("add", add)
worker.register_task("pre_schedule", pre_schedule)
worker.register_task("row_calculator", row_calculator)
worker.register_task("row_aggregator", row_aggregator)
worker.register_task("echo", echo)
print "working"
#print dir(worker)
#print worker.worker_abilities
worker.work()
def task_backtest(gearman_worker, gearman_job):
symbol = ['000001', '603993']
bars = bindata.BackTestData(bindata.raw)
# Apply our current strategy on the chosen stock pool
rfs = CurrentStrategy(symbol, bars)
# specify constraints, here is the default one
cons = Constraint()
# specify a naive optimizer
opt = NaiveOptimizer(cons)
data = json.loads(gearman_job.data)
function_list = {}
signal_generator = compile(data["code"], '', 'exec')
exec signal_generator in function_list
# Create a portfolio
portfolio = MarketOnClosePortfolio(symbol, bars, rfs, \
opt, initial_capital=1000000.0)
portfolio.strategy.sig_generator = function_list["generate_signals"]
# Backtest our portfolio and store result in book
book = portfolio.backtest_portfolio(worker=gearman_worker, job=gearman_job)
ret = book.nav_to_json()
return json.dumps(ret)
if __name__ == "__main__":
# gm_worker.set_client_id is optional
gm_worker.set_client_id('python-worker')
gm_worker.register_task('backtest', task_backtest)
# Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity
gm_worker.work()
