Ejemplo n.º 1
0
    def _setup_workers_files(cls):
        cls.workers_path = "/tmp/wait_jobs_test_workers"
        if not os.path.exists(cls.workers_path):
            os.makedirs(cls.workers_path)

        #WORKER generate_numbers
        open(cls.workers_path+"/generate_numbers.py", 'w').write("""import time
import poliglo
import os

POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'generate_numbers'

def process(specific_info, data, *args):
    inputs = poliglo.get_inputs(data, specific_info)
    numbers_range = inputs.get('numbers_range')
    sleep_time = inputs.get('sleep')

    for i in range(numbers_range[0], numbers_range[1]):
        time.sleep(sleep_time)
        yield {'number': i}

poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
""")

        #WORKER filter
        open(cls.workers_path+"/filter.py", 'w').write("""import time
import poliglo
import os

POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'filter'

def process(specific_info, data, *args):
    inputs = poliglo.get_inputs(data, specific_info)
    min_value = inputs.get("min")
    if inputs['number'] < min_value:
        return [inputs,]
    return []

poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
""")

        #WORKER wait_jobs
        copyfile(os.path.abspath(__file__), cls.workers_path+'/wait_jobs.py')

        #WORKER count_numbers
        open(cls.workers_path+"/count_numbers.py", 'w').write("""import time
import poliglo
import os

POLIGLO_SERVER_URL = os.environ.get('POLIGLO_SERVER_URL')
WORKER_TYPE = 'count_numbers'

def process(specific_info, data, *args):
    connection = args[0].get('connection')

    inputs = poliglo.get_inputs(data, specific_info)
    queue = inputs.get('__read_from_queue')

    total = 0
    while True:
        queue_data = connection.lpop(queue)
        if queue_data is None:
            break
        total +=1
    return [{'total': total}]

config = poliglo.get_config(POLIGLO_SERVER_URL, 'all')
connection = poliglo.get_connection(config)
poliglo.default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection})
""")
def main():
    config = get_config(POLIGLO_SERVER_URL, 'all')
    connection = get_connection(config)
    default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'connection': connection})
Ejemplo n.º 3
0
def main():
    default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process)
def main():
    config = get_config(POLIGLO_SERVER_URL, WORKER_TYPE)
    mongo_connection = MongoClient(config.get('MONGO_URI'))[config.get('MONGO_DB')]

    default_main(POLIGLO_SERVER_URL, WORKER_TYPE, process, {'mongo_connection': mongo_connection})