def main_process(my_id): config_params = ConfigReader().parse_vars( ["RECV_QUEUE", "SEND_QUEUE", "MASTER_SEND_QUEUE", "SEND_REQUEST_QUEUE", "STATUS_QUEUE", "DATA_CLUSTER_WRITE", "DATA_CLUSTER_READ" ] ) worker = Worker( config_params["RECV_QUEUE"], [config_params["SEND_QUEUE"]], config_params["MASTER_SEND_QUEUE"], config_params["SEND_REQUEST_QUEUE"], config_params["STATUS_QUEUE"], config_params["DATA_CLUSTER_WRITE"], config_params["DATA_CLUSTER_READ"], my_id ) while True: worker.start()
def test_celery(): # g1 = group(add.si(2, 3)) # g2 = group([add.si(4, 4)]) # s1 = chord(g1, dummy.si()) # s2 = chord(g2, dummy.si()) # # func = chain([g1, g2]) | final_callback.s() # func = s1 | s2 | final_callback.s() # res = func() # print(res) data = [ '1', '2', '3', '4', ] * 1 info = { 'celery_worker': 'test.functional.test_celery_group_little.simple', 'worker': 'test.functional.test_celery_group_little.worker_do_sth', 'celery_max_workers': 1, 'celery_chunk_size': 2, 'chunk_size': 2, 'queue': 'a1', 'dummy': 'test.functional.test_celery.dummy', } # resp = work(data=data, worker = Worker(mode='celery') resp = worker.work(data, info) return resp
def main(): config_params = ConfigReader().parse_vars( ["RECV_QUEUE", "SEND_QUEUE", "MASTER_SEND_QUEUE"]) worker = Worker(config_params["RECV_QUEUE"], [config_params["SEND_QUEUE"]], config_params["MASTER_SEND_QUEUE"]) worker.start()
def test(self, data, info): print(data, '1111') print(info, '1111') sub_info = deepcopy(info) sub_info['chunk_size'] = info['sub_chunk_size'] sub_info['worker'] = info['sub_worker'] worker = Worker(mode='thread') resp = worker.work(data, sub_info)
def main_process(): config_params = ConfigReader().parse_vars([ "RECV_QUEUE", "SEND_QUEUE", "STATUS_QUEUE", "DATA_CLUSTER_WRITE", "DATA_CLUSTER_READ" ]) worker = Worker(config_params["RECV_QUEUE"], config_params["SEND_QUEUE"], config_params["STATUS_QUEUE"], config_params["DATA_CLUSTER_WRITE"], config_params["DATA_CLUSTER_READ"]) worker.start()
def xtest_it(): print('>>>>') data = [ 'u11', 'u22', 'u33', 'u44', 'u21', 'u22', 'u23', 'u24', 'u31', 'u32', 'u33', 'u34', 'u41', 'u42', 'u43', 'u44', 'u51', 'u52', 'u53', 'u54', ] * 2 info = { 'worker': 'test.functional.test_worker_coroutine.worker_do_sth', 'chunk_size': 4, } worker = Worker(mode='coroutine') resp = worker.work(data, info) print(resp)
def create_app(): log = get_logger(__name__) log.info('Setting up worker...') WORKER_ID = os.getenv('WORKER_ID') MAIN_SERVER_URL = os.getenv('MAIN_SERVER_URL') WORKER_HOST = os.getenv('WORKER_HOST') WORKER_PORT = os.getenv('WORKER_PORT') log.info(""" Parameters: WORKER_ID: {} MAIN_SERVER_URL: {}, WORKER_HOST: {}, WORKER_PORT: {} """.format(WORKER_ID, MAIN_SERVER_URL, WORKER_HOST, WORKER_PORT)) worker = Worker(WORKER_ID, WORKER_HOST, WORKER_PORT, LabsRepository(), LabTemplatesRepository()) log.info("Setting up scheduled jobs") setup_scheduler(worker) log.info("Setting up api") app = Flask('VMs worker') api = Api(app) register_endpoints(api, worker) return app
def work(): customer_queue = queue.Queue() done = event.Event() customers = [] workers = [] jobs = [] for i in range(CONFIG["workers"]): worker = Worker() workers.append(worker) queue_stats = QueueStats(workers, customers) for i in range(CONFIG["workers"]): w = gevent.spawn(workers[i].execute_job, customer_queue, done, queue_stats) jobs.append(w) customer_job = gevent.spawn(insert_new_customer_to_queue, customer_queue, customers, done) jobs.append(customer_job) gevent.joinall(jobs) queue_stats.report()
def _setup_workers(self, num_workers): workers = [] for i in range(0, self.num_workers): workers.append( Worker( module_name=self.module_name, module_args=self.module_args ) ) return workers
def run_remote(self, url, data, callback_data=None): self.worker = Worker(mode=self.info.get('mode', 'thread')) triggers = self.info.get('triggers') trigger_payload = {'trigger': 'netboy.support.triggers.post_it'} if callback_data: trigger_payload.update(callback_data) if triggers: self.info['triggers'].append(trigger_payload) else: self.info['triggers'] = [trigger_payload] payload = { 'url': url, 'method': 'post', 'postfields': { 'info': copy(self.info), 'data': data } } resp = curl_work(payload, logger='netboy') return resp
def xtest_celery(): # g1 = group(add.si(2, 3)) # g2 = group([add.si(4, 4)]) # s1 = chord(g1, dummy.si()) # s2 = chord(g2, dummy.si()) # # func = chain([g1, g2]) | final_callback.s() # func = s1 | s2 | final_callback.s() # res = func() # print(res) data = [ 'u11', 'u22', 'u33', 'u44', 'u21', 'u22', 'u23', 'u24', 'u31', 'u32', 'u33', 'u34', 'u41', 'u42', 'u43', 'u44', 'u51', 'u52', 'u53', 'u54', ]*1 info = { 'celery_worker': 'test.functional.test_celery.simple', 'worker': 'test.functional.test_celery.worker_do_sth', 'celery_max_workers': 1, 'celery_chunk_size': 40, 'chunk_size': 40, # 'final_callback': 'test.functional.test_celery.final_callback_si', 'dummy': 'test.functional.test_celery.dummy', 'sync_callback': 'test.functional.test_celery.final_callback', 'each_callback': 'test.functional.test_celery.callback', # 'queue': 'worker' } # resp = work(data=data, worker = Worker(mode='celery') resp = worker.work(data, info) return resp
def start_labs_job(worker: Worker): log = get_logger(__name__) log.info('Starting start labs job') for lab in worker.labs(): lab_status = lab.status(include_machines=False) status = LabStatus[lab_status.get('status')] if status != LabStatus.PREPARING: continue start_date_str = lab_status['start_date'] start_date = None try: start_date = parse_date(start_date_str) except: log.warning('Lab with id {} has invalid start date'.format(lab.id)) continue if start_date is not None: if datetime.datetime.now() > start_date: log.info('Starting lab {}'.format(lab.id)) lab.start() log.info('Start labs job ended')
from worker.settings import NEW_TASK_EVENT from worker.worker import Worker WORKER_LIST = [Worker(NEW_TASK_EVENT)] # map(lambda x: x.start(), WORKER_LIST) # for x in WORKER_LIST: # x.start()
def work_do_sth(data, info): for i in data: get_picture(i) if __name__ == "__main__": # a = get_picture() # set(a) a = time.time() data = [1, 2, 3, 4, 5, 6] info = {'worker': 'pachong.work_do_sth', 'chunk_size': 2} ww = Worker(mode="thread") resp = ww.work(data, info) # 直接爬取 # for i in data: # get_picture(i) # for i in list(range(1,7)): # th1 = threading.Thread(target=get_picture, args=(i,)) # th1.start() # th1 = threading.Thread(target=get_picture, args=(1,)) # # th1.start() # th2 = threading.Thread(target=get_picture, args=(2,)) # th2.start()
class NetBoy: def __init__(self, info=None): self.info = info if info else {} self.info['dummy'] = 'netboy.celery.tasks.dummy' self.info['log'] = 'netboy' def use_socks5_proxy(self, proxy): p = proxy.split(':') self.info['proxytype'] = 'socks5' self.info['proxy'] = p[0] self.info['proxyport'] = int(p[1]) return self def use_http_proxy(self, proxy): p = proxy.split(':') self.info['proxytype'] = 'http' self.info['proxy'] = p[0] self.info['proxyport'] = int(p[1]) return self def use_queue(self, queue): self.info['queue'] = queue return self def use_logger(self, log_name): self.info['log'] = log_name return self def use_filter(self, result_filter): self.info['filter'] = result_filter return self def use_prepares(self, prepares): self.info['prepares'] = prepares return self def use_triggers(self, triggers): self.info['triggers'] = triggers return self def use_analysers(self, analysers): self.info['analysers'] = analysers return self def use_auth(self, user, password, group='default'): self.info['auth'] = { 'user': user, 'password': password, 'group': group } return self def use_useragent(self, useragent): self.info['useragent'] = useragent return self def use_timeout(self, timeout=None, connect=None, wait=None, script=None): if timeout: self.info['timeout'] = timeout if connect: self.info['connecttimeout'] = connect if wait: self.info['wait'] = wait if script: self.info['script_timeout'] = script return self def use_info(self, info): self.info = info return self def use_final(self, final): self.info['final'] = final return self def use_mode(self, mode): self.info['mode'] = mode spider = self.info.get('spider') if spider: self.use_spider(spider) else: if mode == 'coroutine': self.use_spider('aiohttp') return self def use_spider(self, spider='pycurl'): self.info['spider'] = spider mode = self.info.get('mode', 'thread') if mode == 'celery': if spider == 'pycurl': self.info[ 'celery_worker'] = 'netboy.celery.tasks.pycurl_worker' self.info[ 'worker'] = 'netboy.celery.tasks.multicurl_worker_do_crawl' self.info[ 'final_callback'] = 'netboy.celery.tasks.final_callback' elif spider == 'chrome': self.info[ 'celery_worker'] = 'netboy.celery.tasks.thread_worker' self.info[ 'worker'] = 'netboy.celery.tasks.chrome_worker_do_crawl' self.info[ 'final_callback'] = 'netboy.celery.tasks.final_callback' elif mode == 'coroutine' and spider == 'aiohttp': self.info[ 'worker'] = 'netboy.aio_http.aiohttp_handler.aiohttp_handler' else: if spider == 'pycurl': self.info[ 'worker'] = 'netboy.multi_pycurl.multicurl_handler.curl_handler' elif spider == 'chrome': self.info[ 'worker'] = 'netboy.selenium_chrome.chrome_driver_handler.chrome_driver_handler' elif spider == 'aiohttp': self.info[ 'worker'] = 'netboy.aio_http.aiohttp_handler.aio_http_handler' return self def use_workers(self, workers=8, chunk_size1=40, chunk_size2=8): self.info['celery_max_workers'] = workers self.info['max_workers'] = workers self.info['celery_chunk_size'] = max(chunk_size1, chunk_size2) self.info['chunk_size'] = min(chunk_size1, chunk_size2) return self def use_logger(self, logger): self.info['log'] = logger return self def use_chrome(self, chrome): self.info['chrome'] = chrome self.use_spider('chrome') return self def use_window(self, window): self.info['window_size'] = window self.use_spider('chrome') return self def run(self, data): self.worker = Worker(mode=self.info.get('mode', 'thread')) resp = self.worker.work(data, self.info) return resp def run_remote(self, url, data, callback_data=None): self.worker = Worker(mode=self.info.get('mode', 'thread')) triggers = self.info.get('triggers') trigger_payload = {'trigger': 'netboy.support.triggers.post_it'} if callback_data: trigger_payload.update(callback_data) if triggers: self.info['triggers'].append(trigger_payload) else: self.info['triggers'] = [trigger_payload] payload = { 'url': url, 'method': 'post', 'postfields': { 'info': copy(self.info), 'data': data } } resp = curl_work(payload, logger='netboy') return resp def register_remote(self, url, user, password, group='default'): payload = { 'url': url, 'method': 'post', 'postfields': { 'user': user, 'password': password, 'group': group } } resp = curl_work(payload, logger='netboy') return resp
def init_api(api, app): api_worker = Worker(app.config.get("APP_CONFIG", {}), overide_logger=app.logger) api.workder = api_worker pass
'netboy.celery.tasks.pycurl_worker', 'worker': 'netboy.celery.tasks.multicurl_worker_do_crawl', 'celery_max_workers': 4, 'celery_chunk_size': 10, 'chunk_size': 5, 'queue': 'worker', 'dummy': 'netboy.celery.tasks.dummy', 'filter': ['url', 'cookielist'], 'triggers': [ { 'hello': 'world' }, { 'hello2': 'world2' }, { 'trigger': 'netboy.support.trigger.trig_it' }, ], 'analysers': ['netboy.support.analysers.analyse_it'] } worker = Worker(mode='celery') resp = worker.work(data, info) print(resp)
eel.initJs(width) # Call a Javascript function def webServer(): eel.start('main.html', block=False) # Start (this blocks and enters loop) while True: eel.sleep(1.0) # Use eel.sleep(), not time.sleep() try: _thread.start_new_thread(webServer, ()) # launch the server in parallel print("webServer started !") except: print("Error: unable to start thread") if __name__ == "__main__": # We create our senders maze = PyramidalMaze(width) scout1 = Scout("*****@*****.**", "woweygiowa96") engineer1 = Engineer("*****@*****.**", "woweygiowa96") worker1 = Worker("*****@*****.**", "woweygiowa96") scout1.constructor(maze, eel) scout1.start() engineer1.constructor(maze, eel) engineer1.start() worker1.constructor(maze, eel) worker1.start()
def worker(session, redis, wf): worker = Worker(redis=redis, workflow=wf, session=session) return worker
def main(): worker = Worker("./config/") worker.startWork()
from flask import Flask from flask import render_template import logging from utils.config import Config from utils.common import create_folder, get_folder, init_usign app = Flask(__name__) import server.views config = Config() create_folder("{}/{}".format(get_folder("downloaddir"), "faillogs")) if config.get("sign_images"): print("sign images") init_usign() if config.get("dev"): from worker.worker import Worker worker = Worker() worker.start() #app.debug = True
def main(config): log.init(config.get('worker', 'log_level')) w = Worker(config.get('worker', 'server_at')) w.run()
def run(self, data): self.worker = Worker(mode=self.info.get('mode', 'thread')) resp = self.worker.work(data, self.info) return resp
def add_worker(self, worker_id, worker_type): self.workers.append(Worker(worker_id, worker_type))
def remove_worker(self, worker_id): worker = self.workers.remove(Worker(worker_id, worker_type)) return worker
#!/usr/bin/env python3 import signal import logging from worker.worker import Worker logging.basicConfig(level=logging.DEBUG) w = Worker() signal.signal(signal.SIGINT, w.destroy) signal.signal(signal.SIGTERM, w.destroy) w.run()