def write_finalized_job(workflow_instance_data, worker_output_data, worker_id, connection):
# prepare_write_output(workflow_instance_data, worker_output_data, worker_id):
new_workflow_instance_data = prepare_write_output(
workflow_instance_data, worker_output_data, worker_id
)
connection.zadd(
REDIS_KEY_INSTANCE_WORKER_FINALIZED_JOBS % (
new_workflow_instance_data['workflow_instance']['workflow'],
new_workflow_instance_data['workflow_instance']['id'], worker_id
),
time(),
to_json(new_workflow_instance_data)
)
connection.lpush(
REDIS_KEY_QUEUE_FINALIZED,
to_json({
'workflow': new_workflow_instance_data['workflow_instance']['workflow'],
'workflow_instance_id': new_workflow_instance_data['workflow_instance']['id'],
'workflow_instance_name': new_workflow_instance_data['workflow_instance'].get(
'name', 'untitled'),
# TODO: be able to always return a valid meta_worker
'meta_worker': new_workflow_instance_data['workflow_instance'].get(
'meta_worker', 'not sure'
),
'worker_id': worker_id
})
)
def setUpClass(cls):
cls.test_base_path = '/tmp/tests_poliglo_server'
cls.config_path = _write_file(cls.test_base_path, 'config.json',
to_json(CONFIG))
cls.workflows_path = os.path.join(cls.test_base_path, 'workflows')
for workflow in WORKFLOWS:
_write_file(cls.workflows_path,
'workflow_' + workflow.get('id') + '.json',
to_json(workflow))
poliglo_server.CONFIG = poliglo_server.load_config(cls.config_path)
poliglo_server.DEBUG = True
poliglo_server.WORKFLOWS = poliglo_server.load_workflows(
cls.workflows_path)
cls.app = poliglo_server.app.test_client()
def test_get_one_workflow_instance(self):
response = self.app.post('/workflows/workflow_1/workflow_instances',
data=to_json({'name': 'Script 1 - 1'}),
headers={'content-type': 'application/json'})
workflow_instance_id = json_loads(response.data).get('id')
response = self.app.get('/workflow_instances/' + workflow_instance_id)
workflow_instance = json_loads(response.data)
self.assertEqual('Script 1 - 1', workflow_instance.get('name'))
def process(specific_info, data, *args):
inputs = poliglo.get_inputs(data, specific_info)
connection = args[0].get('connection')
waiting_queue_name = get_waiting_queue_name(
data['process']['id'], data['process']['worker_id'], inputs['wait_jobs_from']
)
connection.zadd(waiting_queue_name, time(), to_json(data))
return []
def test_get_workflow_instance_status_pending(self):
response = self.app.post(
'/workflows/workflow_1/workflow_instances', data=to_json({'name': 'Script 1 - 1'}),
headers={'content-type':'application/json'}
)
workflow_instance_id = json_loads(response.data).get('id')
response = self.app.get('/workflow_instances/'+workflow_instance_id+'/status')
workflow_instance = json_loads(response.data)
self.assertEqual('pending', workflow_instance.get('status'))
def test_workflow_get_workflow_instance_one_exists(self):
url = '/workflows/workflow_1/workflow_instances'
self.app.post(url,
data=to_json({'name': 'Script 1 - 1'}),
headers={'content-type': 'application/json'})
response = self.app.get(url)
workflow_instances = json_loads(response.data)
self.assertEqual(1, len(workflow_instances))
self.assertEqual('Script 1 - 1', workflow_instances[0].get('name'))
def _setup_config(cls):
cls.config = {
"all": {
"REDIS_HOST": "127.0.0.1",
"REDIS_PORT": 6379,
"REDIS_DB": 5,
"POLIGLO_SERVER_URL": "http://localhost:9016"
}
}
cls.config_path = "/tmp/config.json"
open(cls.config_path, 'w').write(to_json(cls.config))
def get_workflow_instance_outputs(workflow_instance_id):
connection = get_connection(CONFIG.get('all'))
workflow_instance_data = _get_workflow_instance(connection, workflow_instance_id)
workflow_id = workflow_instance_data.get('type')
workflow = _get_workflow(workflow_id)
# TODO: Manage multiple final nodes
worker_id = _find_last_worker_id(workflow, workflow.get('start_worker_id'))[0]
target_key = REDIS_KEY_INSTANCE_WORKER_FINALIZED_JOBS % (workflow_id, workflow_instance_id, worker_id)
return to_json([
json_loads(data).get('workers_output', {}).get(worker_id) for data in connection.zrange(target_key, 0, -1)
])
def test_workflow_get_workflow_instance_one_exists(self):
url = '/workflows/workflow_1/workflow_instances'
self.app.post(
url, data=to_json({'name': 'Script 1 - 1'}),
headers={'content-type':'application/json'}
)
response = self.app.get(url)
workflow_instances = json_loads(response.data)
self.assertEqual(1, len(workflow_instances))
self.assertEqual('Script 1 - 1', workflow_instances[0].get('name'))
def setUpClass(cls):
cls.test_base_path = '/tmp/tests_poliglo_server'
cls.config_path = _write_file(
cls.test_base_path,
'config.json',
to_json(CONFIG)
)
cls.workflows_path = os.path.join(cls.test_base_path, 'workflows')
for workflow in WORKFLOWS:
_write_file(
cls.workflows_path,
'workflow_'+workflow.get('id')+'.json',
to_json(workflow)
)
poliglo_server.CONFIG = poliglo_server.load_config(cls.config_path)
poliglo_server.DEBUG = True
poliglo_server.WORKFLOWS = poliglo_server.load_workflows(
cls.workflows_path
)
cls.app = poliglo_server.app.test_client()
def write_one_output(connection, output_meta_worker, output_worker_id, workflow_instance_data):
new_job_id = str(uuid4())
workflow_instance_data['jobs_ids'] = workflow_instance_data['jobs_ids'] + [new_job_id]
workflow_instance_data['workflow_instance']['worker_id'] = output_worker_id
workflow_instance_data['workflow_instance']['meta_worker'] = output_meta_worker
add_new_job_id(
connection,
workflow_instance_data['workflow_instance']['workflow'],
workflow_instance_data['workflow_instance']['id'],
output_worker_id,
new_job_id
)
add_data_to_next_worker(connection, output_meta_worker, to_json(workflow_instance_data))
def _setup_script(cls):
script = {
"id": "test_wait_jobs",
"name": "test_wait_jobs",
"start_worker_id": "generate_numbers_1",
"workers": [
{
"worker_type": "generate_numbers",
"id": "generate_numbers_1",
"default_inputs": {
"numbers_range": [0, 10],
"sleep": 0
},
"next_workers": ["filter_1"]
},
{
"worker_type": "filter",
"id": "filter_1",
"default_inputs": {
"min": 1000
},
"next_workers": ["wait_jobs_1"]
},
{
"worker_type": "wait_jobs",
"id": "wait_jobs_1",
"default_inputs": {
"wait_jobs_from": ["generate_numbers_1", "filter_1", "wait_jobs_1"]
},
"next_workers": ["count_numbers_1"]
},
{
"id": "count_numbers_1",
"worker_type": "count_numbers",
"next_workers": []
}
]
}
cls.script_path = "/tmp/wait_jobs_test_scripts"
if not os.path.exists(cls.script_path):
os.makedirs(cls.script_path)
open(cls.script_path+"/script_test_wait_jobs.json", 'w').write(to_json(script))
def _setup_workflow(cls):
workflow = {
"id": "test_wait_jobs",
"name": "test_wait_jobs",
"start_worker_id": "generate_numbers_1",
"workers": {
"generate_numbers_1": {
"meta_worker": "generate_numbers",
"default_inputs": {
"numbers_range": [0, 10],
"sleep": 0
},
"next_workers": ["filter_1"]
},
"filter_1": {
"meta_worker": "filter",
"default_inputs": {
"min": 1000
},
"next_workers": ["wait_jobs_1"]
},
"wait_jobs_1": {
"meta_worker": "wait_jobs",
"default_inputs": {
"wait_jobs_from": ["generate_numbers_1", "filter_1", "wait_jobs_1"]
},
"next_workers": ["count_numbers_1"]
},
"count_numbers_1": {
"meta_worker": "count_numbers",
"next_workers": []
}
}
}
cls.workflow_path = "/tmp/wait_jobs_test_workflows"
if not os.path.exists(cls.workflow_path):
os.makedirs(cls.workflow_path)
open(cls.workflow_path+"/workflow_test_wait_jobs.json", 'w').write(to_json(workflow))
def stats_add_new_instance(connection, workflow, workflow_instance_info):
connection.zadd(REDIS_KEY_INSTANCES % workflow, time(),
to_json(workflow_instance_info))
})
)
def write_error_job(connection, worker_id, raw_data, error):
metric_name = 'errors'
try:
workflow_instance_data = json_loads(raw_data)
if not workflow_instance_data.get('workers_error'):
workflow_instance_data['workers_error'] = {}
workflow_instance_data['workers_error'][worker_id] = {
'error': str(error), 'traceback': traceback.format_exc()
}
metric_name = REDIS_KEY_INSTANCE_WORKER_ERRORS % (
workflow_instance_data['workflow_instance']['workflow'],
workflow_instance_data['workflow_instance']['id'],
worker_id
)
except Exception, e:
workflow_instance_data = {'workers_error': {}, 'raw_data': raw_data}
workflow_instance_data['workers_error'][worker_id] = {
'error': 'cannot json_loads', 'traceback': traceback.format_exc()
}
metric_name = REDIS_KEY_INSTANCE_WORKER_ERRORS % (
'unknown', 'unknown', worker_id
)
try:
json_encoded = to_json(workflow_instance_data)
except Exception, e:
json_encoded = to_json(convert_object_to_unicode(workflow_instance_data))
connection.zadd(metric_name, time(), json_encoded)
