Ejemplo n.º 1
0
    def _init(self):
        while True:
            while not self.is_available():
                logging.error(
                    "Gearman not available right now. Demon will sleep during {n} seconds"
                    .format(n=settings.GEARMAN_RECONNECT_TIMEOUT))
                gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT)

            logging.debug("Gearman worker try to connect {hosts}".format(
                hosts=', '.join(settings.GEARMAN['hosts'])))

            try:
                gm_worker = GearmanWorker(settings.GEARMAN['hosts'])
                gm_worker.set_client_id("socket_io_gearman_" +
                                        str(OPTIONS.port))
                gm_worker.register_task("socket_io", GearmanListener.callback)
                logging.debug("Gearman worker was successfull created")

                return gm_worker
            except Exception, e:
                logging.error(
                    "Error while initiation gearman worker connect with message: {message}"
                    .format(message=e.message))
                logging.debug("Demon will be sleep during {n} seconds".format(
                    n=settings.GEARMAN_RECONNECT_TIMEOUT))
                gevent.sleep(settings.GEARMAN_RECONNECT_TIMEOUT)
Ejemplo n.º 2
0
        myj = Jenkins(jenkins_data['url'])
        job = myj.get_job(jenkins_data['job_id'])
        #job.invoke(securitytoken=token, block=block)
        job.invoke(invoke_pre_check_delay=0)
    except:
        rev = "Not Happy!!!"

    return rev


# Establish a connection with the job server on localhost--like the client,
# multiple job servers can be used.
worker = GearmanWorker(['localhost'])

# register_task will tell the job server that this worker handles the "echo"
# task
worker.set_client_id('your_worker_client_id_name')
worker.register_task('echo', task_listener_echo)
worker.register_task('build:pep8', task_listener_build)
worker.register_task('stop:jenkins_master.hp.com', task_listener_stop)
worker.register_task('bravo', task_listener_echo)
worker.register_task('reverse', task_listener_reverse)
worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job)

# Once setup is complete, begin working by consuming any tasks available
# from the job server
print 'working...'
worker.work()

# The worker will continue to run (waiting for new work) until exited by
# code or an external signal is caught
Ejemplo n.º 3
0
        #TODO: Log the error
        return str(e)


################################################


def task_listener_email(gearman_worker, gearman_job):
    #TODO: Log the call and params
    cleaned_words, reply, meta, infile, sample_text = process_words(
        gearman_job.data)
    #TODO: Log the call and params
    res = inverted_index(cleaned_words, reply, meta, infile, sample_text)
    print res
    #TODO: Log the result
    if res != 'Ok':
        # Log the stack trace and exception
        return 'NotOk'
    else:
        return res


if __name__ == "__main__":
    # TODO:  Daemonize
    # TODO: Ensure there are not multiple monitors for the same location (check for lock file)
    # TODO: Make the strings config driven
    # TODO: Have a meaningful try-except block
    gm_worker.set_client_id('inv-index-worker')
    gm_worker.register_task('invindex', task_listener_email)
    gm_worker.work()
Ejemplo n.º 4
0
def task_backtest(gearman_worker, gearman_job):
    symbol = ['000001', '603993']
    bars = bindata.BackTestData(bindata.raw)
    # Apply our current strategy on the chosen stock pool
    rfs = CurrentStrategy(symbol, bars)
    # specify constraints, here is the default one
    cons = Constraint()
    # specify a naive optimizer
    opt = NaiveOptimizer(cons)

    data = json.loads(gearman_job.data)
    function_list = {}
    signal_generator = compile(data["code"], '', 'exec')
    exec signal_generator in function_list

    # Create a portfolio
    portfolio = MarketOnClosePortfolio(symbol, bars, rfs, \
                opt, initial_capital=1000000.0)
    portfolio.strategy.sig_generator = function_list["generate_signals"]
    # Backtest our portfolio and store result in book
    book = portfolio.backtest_portfolio(worker=gearman_worker, job=gearman_job)
    ret = book.nav_to_json()
    return json.dumps(ret)


if __name__ == "__main__":
    # gm_worker.set_client_id is optional
    gm_worker.set_client_id('python-worker')
    gm_worker.register_task('backtest', task_backtest)
    # Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity
    gm_worker.work()
Ejemplo n.º 5
0
    topN_list = [{item['word_doc_hash'].split('_')[1]:item} for item in topN]
    
    # NOTE: Persist topN into redis
    redis_server.set(current_topn_pattern, str(topN_list))

    # NOTE: remove the tbp_<word> from the to be processed list
    redis_server.delete(word) 
    return 'Ok'   

########################################################################


def task_listener_topncompute(gearman_worker, gearman_job):
    #TODO: Log the call and params
    res = topncompute(gearman_job.data)
    print res
    #TODO: Log the result
    if res != 'Ok':
        # Log the stack trace and exception
        return 'NotOk'
    else:
        return res

if __name__ == "__main__":
    # TODO:  Daemonize 
    # TODO: Ensure there are not multiple monitors for the same location (check for lock file)
    # TODO: Make the strings config driven
    # TODO: Have a meaningful try-except block
    gm_worker.set_client_id('topn-worker')
    gm_worker.register_task('topncompute', task_listener_topncompute)
    gm_worker.work()
        myj = Jenkins(jenkins_data['url'])
        job = myj.get_job(jenkins_data['job_id'])
        #job.invoke(securitytoken=token, block=block)
        job.invoke(invoke_pre_check_delay=0)
    except:
        rev="Not Happy!!!" 
    
    return rev

	
# Establish a connection with the job server on localhost--like the client,
# multiple job servers can be used.
worker = GearmanWorker(['localhost'])

# register_task will tell the job server that this worker handles the "echo"
# task
worker.set_client_id('your_worker_client_id_name')
worker.register_task('echo', task_listener_echo)
worker.register_task('build:pep8', task_listener_build)
worker.register_task('stop:jenkins_master.hp.com', task_listener_stop)
worker.register_task('bravo', task_listener_echo)
worker.register_task('reverse', task_listener_reverse)
worker.register_task('jenkins_invoke_job', task_listener_jenkins_invoke_job)

# Once setup is complete, begin working by consuming any tasks available
# from the job server
print 'working...'
worker.work()

# The worker will continue to run (waiting for new work) until exited by
# code or an external signal is caught
Ejemplo n.º 7
0
        return self.on_job_exception(current_job, sys.exc_info())
    self.task_count +=1
    if self.task_count >= 50:
        print "max tasks reached. exiting"
        sys.exit()

    return self.on_job_complete(current_job, job_result)


GearmanWorker.task_count = 0
GearmanWorker.on_job_exception = on_job_exception
GearmanWorker.on_job_execute = on_job_execute

worker = GearmanWorker(JOBSERVER_LIST)

worker.set_client_id("working_on_the_djangoroad")

worker.register_task("add", add)
worker.register_task("pre_schedule", pre_schedule)
worker.register_task("row_calculator", row_calculator)
worker.register_task("row_aggregator", row_aggregator)

worker.register_task("echo", echo)


print "working"
#print dir(worker)
#print worker.worker_abilities
worker.work()

Ejemplo n.º 8
0
def task_backtest(gearman_worker, gearman_job):
    symbol = ['000001', '603993']
    bars = bindata.BackTestData(bindata.raw)
    # Apply our current strategy on the chosen stock pool
    rfs = CurrentStrategy(symbol, bars)
    # specify constraints, here is the default one
    cons = Constraint()
    # specify a naive optimizer
    opt = NaiveOptimizer(cons)

    data = json.loads(gearman_job.data)
    function_list = {}
    signal_generator = compile(data["code"], '', 'exec')
    exec signal_generator in function_list

    # Create a portfolio
    portfolio = MarketOnClosePortfolio(symbol, bars, rfs, \
                opt, initial_capital=1000000.0)
    portfolio.strategy.sig_generator = function_list["generate_signals"]
    # Backtest our portfolio and store result in book
    book = portfolio.backtest_portfolio(worker=gearman_worker, job=gearman_job)
    ret = book.nav_to_json()
    return json.dumps(ret)


if __name__ == "__main__":
    # gm_worker.set_client_id is optional
    gm_worker.set_client_id('python-worker')
    gm_worker.register_task('backtest', task_backtest)
    # Enter our work loop and call gm_worker.after_poll() after each time we timeout/see socket activity
    gm_worker.work()