def _setup_workers_files(cls): cls.workers_path = "/tmp/wait_jobs_test_workers" if not os.path.exists(cls.workers_path): os.makedirs(cls.workers_path) #WORKER generate_numbers open(cls.workers_path+"/generate_numbers.py", 'w').write("""import time import poliglo import os POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL') WORKER_TYPE = 'generate_numbers' def process(specific_info, data, *args): inputs = poliglo.get_inputs(data, specific_info) numbers_range = inputs.get('numbers_range') sleep_time = inputs.get('sleep') for i in range(numbers_range[0], numbers_range[1]): time.sleep(sleep_time) yield {'number': i} poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process) """) #WORKER filter open(cls.workers_path+"/filter.py", 'w').write("""import time import poliglo import os POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL') WORKER_TYPE = 'filter' def process(specific_info, data, *args): inputs = poliglo.get_inputs(data, specific_info) min_value = inputs.get("min") if inputs['number'] < min_value: return [inputs,] return [] poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process) """) #WORKER wait_jobs copyfile(os.path.abspath(__file__), cls.workers_path+'/wait_jobs.py') #WORKER count_numbers open(cls.workers_path+"/count_numbers.py", 'w').write("""import time import poliglo import os POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL') WORKER_TYPE = 'count_numbers' def process(specific_info, data, *args): connection = args[0].get('connection') inputs = poliglo.get_inputs(data, specific_info) queue = inputs.get('__read_from_queue') total = 0 while True: queue_data = connection.lpop(queue) if queue_data is None: break total +=1 return [{'total': total}] config = poliglo.get_config(POLIGLO_SERVER_URL, 'all') connection = poliglo.get_connection(config) poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection}) """)
def main(): config = get_config(POLIGLO_SERVER_URL, 'all') connection = get_connection(config) default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection})
def main(): default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
def main(): config = get_config(POLIGLO_SERVER_URL, WORKER_TYPE) mongo_connection = MongoClient(config.get('MONGO_URI'))[config.get('MONGO_DB')] default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'mongo_connection': mongo_connection})