def _setup_workers_files(cls):
cls.workers_path = "/tmp/wait_jobs_test_workers"
if not os.path.exists(cls.workers_path):
os.makedirs(cls.workers_path)
#WORKER generate_numbers
open(cls.workers_path+"/generate_numbers.py", 'w').write("""import time
import poliglo
import os
POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'generate_numbers'
def process(specific_info, data, *args):
inputs = poliglo.get_inputs(data, specific_info)
numbers_range = inputs.get('numbers_range')
sleep_time = inputs.get('sleep')
for i in range(numbers_range[0], numbers_range[1]):
time.sleep(sleep_time)
yield {'number': i}
poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
""")
#WORKER filter
open(cls.workers_path+"/filter.py", 'w').write("""import time
import poliglo
import os
POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'filter'
def process(specific_info, data, *args):
inputs = poliglo.get_inputs(data, specific_info)
min_value = inputs.get("min")
if inputs['number'] < min_value:
return [inputs,]
return []
poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
""")
#WORKER wait_jobs
copyfile(os.path.abspath(__file__), cls.workers_path+'/wait_jobs.py')
#WORKER count_numbers
open(cls.workers_path+"/count_numbers.py", 'w').write("""import time
import poliglo
import os
POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'count_numbers'
def process(specific_info, data, *args):
connection = args[0].get('connection')
inputs = poliglo.get_inputs(data, specific_info)
queue = inputs.get('__read_from_queue')
total = 0
while True:
queue_data = connection.lpop(queue)
if queue_data is None:
break
total +=1
return [{'total': total}]
config = poliglo.get_config(POLIGLO_SERVER_URL, 'all')
connection = poliglo.get_connection(config)
poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection})
""")
def main():
config = get_config(POLIGLO_SERVER_URL, 'all')
connection = get_connection(config)
default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection})
def main():
default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
def main():
config = get_config(POLIGLO_SERVER_URL, WORKER_TYPE)
mongo_connection = MongoClient(config.get('MONGO_URI'))[config.get('MONGO_DB')]
default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'mongo_connection': mongo_connection})