def start(self, data):
"""This method runs the algorithm job.
This method will start an algorithm asynchronously.
The data parameter must have the key 'algorithm_id' containing the
algorithm id to be started.
The method will retrieve an algorithm_info_model with the given id
and it will create an algorithm_model and save it. By saving the model
a algorithm_task will start asynchronously.
Args:
self: The object pointer.
data: A dictorianry containing the input data for the job.
Returns:
The job_data attribute. The ok attribute will be True if the algorithm
has started, False otherwise.
@see algorithm_info_model @link avi.models.algorithm_info_model
@see algorithm_model @link avi.models.algorithm_model
@see algorithm_task @link avi.task.algorithm_task
"""
log = logger().get_log("algorithm_task")
log.info("job %s", data)
if "algorithm_id" in data:
mng = algorithm_manager()
if not data.get('algorithm_id'):
self.job_data.ok = False
return self.job_data
if len(data['algorithm_id']) < 1:
self.job_data.ok = False
return self.job_data
alg = algorithm_info_model.objects.get(pk=data['algorithm_id'][0])
result = mng.get_algorithm_data(data['algorithm_id'][0], alg.name,
alg.definition_file, data)
m = algorithm_model(alg_name=alg.name,
params=result,
results=result)
m.save()
self.job_data.data = m
self.job_data.ok = True
return self.job_data
# OLD
m = algorithm_model(alg_name=data['algorithm']['name'],
params=data,
results=data)
m.save()
#log.info("model %s", str(m))
self.job_data.data = m
self.job_data.ok = True
return self.job_data
def test_init(self):
test_log("Testing the initialization method...",self)
test_log("Changing the warehouse paths",self)
wh = warehouse().get()
wh.ALGORITHM_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)) ,'algorithms',
'installed')
wh.UPLOADED_ALGORITHM_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)) ,'algorithms',
'uploaded')
test_log("Testing the initialization with an empty database",self)
mng = algorithm_manager()
mng.init()
alg = algorithm_info_model.objects.filter(name='installed',
algorithm_type='installed')
self.assertEqual(len(alg),1)
alg = algorithm_info_model.objects.filter(name='temp',
algorithm_type='temporal')
self.assertFalse(alg)
alg = algorithm_info_model.objects.filter(name='uploaded',
algorithm_type='uploaded')
self.assertEqual(len(alg),1)
def start(self, data):
"""This method is deprecated"""
log = logger().get_log('views')
log.info(data)
from avi.core.algorithm.algorithm_manager import algorithm_manager
res = algorithm_manager().get_algorithm(data)
self.job_data.data = {}
self.job_data.ok = True
return self.job_data
def test_has_param_type(self):
test_log("Testing the get_info method...",self)
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'algorithms')
mng = algorithm_manager()
test_log("Testing an OK file",self)
ret = mng.has_param_type(path+"/test.json",'float')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'integer')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'long')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'complex')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'string')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'gaia_table')
self.assertTrue(ret)
ret = mng.has_param_type(path+"/test.json",'hsa_table')
self.assertTrue(ret)
test_log("Testing non valid files",self)
ret = mng.has_param_type(path+"/test.py",'float')
self.assertIsNone(ret)
ret = mng.has_param_type(path+"/non_existing_file.json",'float')
self.assertIsNone(ret)
ret = mng.has_param_type(path,'float')
self.assertIsNone(ret)
ret = mng.has_param_type("/non/existing/path/test.json",'float')
self.assertIsNone(ret)
test_log("Testing non valied types",self)
ret = mng.has_param_type(path+"/test.json",'non-existiny-type')
self.assertFalse(ret)
ret = mng.has_param_type(path+"/test.json",None)
self.assertFalse(ret)
def get_algorithm_list(self):
"""Returns the list of algorithms
Args:
self: The object pointer
Returns:
A dictionary with the information of all algorithms
"""
from avi.core.algorithm.algorithm_manager import algorithm_manager
return algorithm_manager().get_algorithm_list()
def get_algorithm(self, data):
"""Returns the algorithm information
Args:
self: The object pointer
data: A JSON file with the information
Returns:
A dictionary with the algorithm information
"""
from avi.core.algorithm.algorithm_manager import algorithm_manager
return algorithm_manager().get_algorithm_info(data)
def test_get_info(self):
test_log("Testing the get_info method...",self)
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'algorithms')
mng = algorithm_manager()
test_log("Testing an OK file",self)
ret = mng.get_info(path+"/test.json",'name')
self.assertEqual(self._def['algorithm']['name'], ret)
self.assertEqual(str(self._def['algorithm']['name']), str(ret))
test_log("Testing non valid files",self)
ret = mng.get_info(path+"/test.py",'name')
self.assertIsNone(ret)
self.assertNotEqual(self._def['algorithm']['name'], ret)
self.assertNotEqual(str(self._def['algorithm']['name']), str(ret))
ret = mng.get_info(path+"/non_existing_file.json",'name')
self.assertIsNone(ret)
self.assertNotEqual(self._def['algorithm']['name'], ret)
self.assertNotEqual(str(self._def['algorithm']['name']), str(ret))
ret = mng.get_info(path,'name')
self.assertIsNone(ret)
self.assertNotEqual(self._def['algorithm']['name'], ret)
self.assertNotEqual(str(self._def['algorithm']['name']), str(ret))
ret = mng.get_info("/non/existing/path/test.json",'name')
self.assertIsNone(ret)
self.assertNotEqual(self._def['algorithm']['name'], ret)
self.assertNotEqual(str(self._def['algorithm']['name']), str(ret))
test_log("Testing non valied keys",self)
ret = mng.get_info(path+"/test.json",'non-existiny-key')
self.assertIsNone(ret)
ret = mng.get_info(path+"/test.json",None)
self.assertIsNone(ret)
def _create_algorithm_info(self):
mng = algorithm_manager()
alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))
,'algorithms', 'installed')
mng.update_database(alg_dir, 'installed')
alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))
,'algorithms', 'temp')
mng.update_database(alg_dir,'temporal')
alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))
,'algorithms', 'uploaded')
mng.update_database(alg_dir,'uploaded')
def test_get_algorithm_data(self):
test_log("Testing the get_info method...",self)
alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))
,'algorithms')
#algorithm_info_model.objects.create(name="test",
# source_file=alg_dir + "/test.py",
# name_view="Test View",
# definition_file=alg_dir +
# "/test.json",
# algorithm_type="installed")
mng = algorithm_manager()
test_log("Testing an OK file and OK post",self)
post = self._post_exec_algorithm()
ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post)
self.assertEqual(sorted(ret), sorted(self._val))
self.assertEqual(str(sorted(ret)), str(sorted(self._val)))
test_log("Testing non valid files",self)
ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.py",post)
self.assertIsNone(ret)
ret = mng.get_algorithm_data('1','test',
alg_dir+"/non_existing_file.json",post)
self.assertIsNone(ret)
ret = mng.get_algorithm_data('1','test', alg_dir,post)
self.assertIsNone(ret)
ret = mng.get_algorithm_data('1','test',"/non/existing/path/test.json",
post)
self.assertIsNone(ret)
test_log("Testing non valid POST",self)
post = None
ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post)
self.assertIsNone(ret)
post = {}
ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post)
self.assertIsNone(ret)
def test_get_algorithm(self):
test_log("Testing the get_algorithm method...",self)
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'algorithms')
mng = algorithm_manager()
test_log("Testing an OK file",self)
ret = mng.get_algorithm(path+"/test.json")
self.assertEqual(self._def, ret)
self.assertEqual(str(self._def), str(ret))
test_log("Testing non valid files",self)
ret = mng.get_algorithm(path+"/test.py")
self.assertIsNone(ret)
self.assertNotEqual(self._def, ret)
self.assertNotEqual(str(self._def), str(ret))
ret = mng.get_algorithm(path+"/non_existing_file.json")
self.assertIsNone(ret)
self.assertNotEqual(self._def, ret)
self.assertNotEqual(str(self._def), str(ret))
ret = mng.get_algorithm(path)
self.assertIsNone(ret)
self.assertNotEqual(self._def, ret)
self.assertNotEqual(str(self._def), str(ret))
ret = mng.get_algorithm("/non/existing/path/test.json")
self.assertIsNone(ret)
self.assertNotEqual(self._def, ret)
self.assertNotEqual(str(self._def), str(ret))
def start(self, data):
"""This method retrieves the algorithm information.
The data parameter must have the key 'id' containing the algorithm id
which info will be retrieved.
The method will retrieve the algorithm_info_model with the given id and
it will will extract the parameters information from the definition
file provided by the algorithm_info_model.
If the algorithm has gaia, herschel or results files in its input, this
method will also retrieve the names of thouse files from the
resource_model.
Args:
self: The object pointer.
data: A dictorianry containing the input data for the job.
Returns:
The job_data attribute. The ok attribute will be True if the algorithm
exists, False otherwise.
@see algorithm_info_model @link avi.models.algorithm_info_model
@see resource_model @link avi.models.resource_model
"""
log = logger().get_log('algorithm_manager')
log.info(data)
am = algorithm_info_model.objects.get(pk=data['id'])
if not am:
self.log_data.ok = False
return self.job_data
alg_info = algorithm_manager().get_algorithm(am.definition_file)
inp = alg_info['algorithm']['input']
#inp = {}
#inp = sorted(inp.items())
#inp = sorted(inp, key = lambda x: (x.get('group') is None, x))
self.job_data.data = dict()
self.job_data.data['algorithm'] = alg_info['algorithm']
inp = []
for _, i in alg_info['algorithm']['input'].items():
log.info(i)
inp.append(i)
inp.sort(key=lambda x: x['position'], reverse=False)
# self.job_data.data['algorithm_input'] = inp
self.job_data.data.update({'algorithm_input':inp})
ms = resource_model.objects.all()
has_gaia = algorithm_manager().has_param_type(am.definition_file,
'gaia_table')
if has_gaia:
# self.job_data.data['gaia'] = []
self.job_data.data.update({'gaia':[]})
has_hsa = algorithm_manager().has_param_type(am.definition_file,
'hsa_table')
if has_hsa:
# self.job_data.data['hsa'] = []
self.job_data.data.update({'hsa':[]})
log.info(am.definition_file)
has_results = algorithm_manager().has_param_type(am.definition_file,
'results_data')
if has_results:
# self.job_data.data['res'] = []
self.job_data.data.update({'res':[]})
has_user = algorithm_manager().has_param_type(am.definition_file,
'user_data')
if has_user:
# self.job_data.data['user'] = []
self.job_data.data.update({'user':[]})
for i in ms:
log.info("file %s - %s", i.name, i.file_type)
if has_gaia and i.file_type == 'gaia':
self.job_data.data['gaia'].extend([i.name])
if has_hsa and i.file_type == 'hsa':
self.job_data.data['hsa'].extend([i.name])
if has_results and i.file_type == 'result':
self.job_data.data['res'].extend([i.name])
if has_user and i.file_type == 'user':
self.job_data.data['user'].extend([i.name])
#self.job_data.data = alg_info
self.job_data.ok = alg_info is not None
return self.job_data
def start(self, data):
"""This method runs the get_algorithms job.
If the algorithms are not loaded it will load them. Then it will
retrieve the all algorithm_info_models and return the data from them.
Args:
self: The object pointer.
data: A dictionary containing the input data for the job.
Returns:
The job_data attribute. The ok attribute will be True if there are
algorithms retrieved, False otherwise.
@see algorithm_info_model @link avi.models.algorithm_info_model
"""
log = logger().get_log('algorithm_manager')
wh_f = wh_frontend_config().get()
if not wh().get().ALGORITHMS_LOADED:
from avi.core.algorithm.algorithm_manager import algorithm_manager
algorithm_manager().init()
wh().get().ALGORITHMS_LOADED = True
from avi.models import algorithm_info_model, algorithm_group_model
all_ms = algorithm_info_model.objects.all().order_by(
'name_view', 'name', 'pk')
#sall_ms = sorted(all_ms, key = lambda x:(x.name_view is None, x))
self.job_data.data = {}
self.job_data.ok = all_ms is not None
if not all_ms:
return self.job_data
pg = Paginator(all_ms, wh_f.MAX_ALG_PER_PAGE)
page = wh_f.CURRENT_ALG_PAGE
if page < 1:
wh_f.CURRENT_ALG_PAGE = 1
elif page > pg.num_pages:
wh_f.CURRENT_ALG_PAGE = pg.num_pages
ms = pg.page(wh_f.CURRENT_ALG_PAGE)
all_ms_g = algorithm_group_model.objects.all().order_by(
'name_view', 'name', 'pk')
pg_g = Paginator(all_ms_g, wh_f.MAX_ALG_PER_PAGE)
page = wh_f.CURRENT_ALG_PAGE
if page < 1:
wh_f.CURRENT_ALG_PAGE = 1
elif page > pg.num_pages:
wh_f.CURRENT_ALG_PAGE = pg.num_pages
ms_g = pg_g.page(wh_f.CURRENT_ALG_PAGE)
data = []
for g in ms_g:
data.append({"group": g, "algorithms": []})
data.sort(key=lambda x: x["group"].position, reverse=False)
for j in ms:
for g in data:
if j.algorithm_group == g["group"].name:
g["algorithms"].append(
(j.pk, j.name, j.name_view, j.algorithm_type,
j.algorithm_group, j.position))
for g in data:
g["algorithms"].sort(key=lambda x: x[4], reverse=False)
for g in data:
log.info(g["group"].name)
log.info(g["group"].position)
for a in g["algorithms"]:
log.info(a[4])
log.info(a[1])
# OLD
# data = {}
# i = 0
# for j in ms:
# data[i] = (j.pk,
# j.name,
# j.name_view,
# j.algorithm_type)
# i += 1
res = {}
res["algorithms"] = data
res["max_pages"] = pg.num_pages
res["current_page"] = wh_f.CURRENT_ALG_PAGE
res["next_page"] = wh_f.CURRENT_ALG_PAGE + 1
res["prev_page"] = wh_f.CURRENT_ALG_PAGE - 1
self.job_data.data = res
return self.job_data
def test_job_abort_start_ok(self):
input_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'input')
alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'algorithms')
test_log("Input path: %s" % (input_dir), self)
test_log("Algorithms path: %s" % (alg_dir), self)
algorithm_info_model.objects.create(name="test",
source_file=alg_dir + "/test.py",
name_view="Test View",
definition_file=alg_dir +
"/test.json",
algorithm_type="installed")
#test_info("Starting...",self)
pk = 1
try:
alg_info = algorithm_info_model.objects.get(pk=pk)
except algorithm_info_model.DoesNotExist:
#test_info("[FAILED]",self)
self.assertTrue(False)
mng = algorithm_manager()
self.assertIsNotNone(mng)
post = self._post_exec_algorithm(pk, alg_info.name, p5="WAIT_ABORT")
alg_input = mng.get_algorithm_data(str(pk), alg_info.name,
alg_info.definition_file, post)
alg_exec = algorithm_model.objects.create(alg_name=alg_info.name,
params=alg_input,
results={})
alg_exec.save()
while alg_exec.request.pipeline_state.state != 'PENDING': #'STARTED':
try:
alg_exec = algorithm_model.objects.get(pk=pk)
except algorithm_model.DoesNotExist:
#test_info("[FAILED]", self)
self.assertTrue(False)
#test_info(alg_exec.request.pipeline_state.state)
time.sleep(1)
status = alg_exec.request.pipeline_state.state
self.assertNotEqual(status, 'FAILURE')
self.assertNotEqual(status, 'SUCCESS')
self.assertFalse(alg_exec.is_aborted)
from avi.core.pipeline.job_abort import abort
res = abort().start(self._post_abort('algorithm', pk))
self.assertTrue(res.ok)
try:
alg_exec = algorithm_model.objects.get(pk=pk)
except algorithm_model.DoesNotExist:
#test_info("[FAILED]", self)
self.assertTrue(False)
#test_info(str(res.data.params))
#test_info(str(alg_exec.params))
#while alg_exec.request.pipeline_state.state != 'FAILURE':
#test_info(alg_exec.request.pipeline_state.state)
# time.sleep(1)
status = alg_exec.request.pipeline_state.state
self.assertEqual(status, 'FAILURE')
self.assertTrue(alg_exec.is_aborted)
def start(self, data):
"""This method runs the get_pipeline_status job.
This method will retrieve the algorithm_models and it will sort them
by the current sorting method provided by the wh_frontend_config
warehouse.
Then it will paginate the results with current page retrieved from
the wh_frontend_config warehouse.
Args:
self: The object pointer.
data: A dictorianry containing the input data for the job.
Returns:
The job_data attribute. The ok attribute provides the pages information.
@see algorithm_model @link avi.models.algorithm_model
@see wh_frontend_config @link avi.warehouse.wh_frontend_config
"""
log = logger().get_log("get_pipeline_status")
wh = wh_frontend_config().get()
sorting_wh = wh.SORTING_EXEC_BY
order_by = 'request__pipeline_state__started_time'
if sorting_wh == 'name':
order_by = 'alg_name'
elif sorting_wh == '-name':
order_by = '-alg_name'
elif sorting_wh == '-date':
order_by = '-request__pipeline_state__started_time'
elif sorting_wh == 'status':
order_by = 'request__pipeline_state__state'
elif sorting_wh == '-status':
order_by = '-request__pipeline_state__state'
all_ms = algorithm_model.objects.all().order_by(order_by, 'pk')
self.job_data.data = {}
self.job_data.ok = all_ms is not None
if not all_ms:
self.job_data.ok = False
return self.job_data
pg = Paginator(all_ms, wh.MAX_EXEC_PER_PAGE)
page = wh.CURRENT_EXEC_PAGE
#------------------
#----all data algorithms
alldata = {}
k = 0
for h in pg.object_list:
try:
status = h.request.pipe_state.state
date = h.request.pipe_state.started_time
except AttributeError:
status = h.request.pipeline_state.state
date = h.request.pipeline_state.started_time
#status = h.request.pipeline_state.state
#date = h.request.pipeline_state.started_time
params = {}
params['algorithm'] = {'name': h.alg_name, 'params': {}}
ainfo_ms = algorithm_info_model.objects.filter(name=h.alg_name)[0]
if ainfo_ms:
qparams = literal_eval(h.params)
mng = algorithm_manager()
ainfo = mng.get_info(ainfo_ms.definition_file, 'input')
for l in ainfo:
if 'view_name' in ainfo[l]:
params['algorithm']['params'][ainfo[l][
'view_name']] = qparams['algorithm']['params'][
ainfo[l]['name']]
else:
params['algorithm']['params'][
ainfo[l]['name']] = qparams['algorithm']['params'][
ainfo[l]['name']]
params = str(params)
else:
params = h.params
alldata[k] = (h.alg_name, h.pk, params, date, status)
log.info(params)
k += 1
#------------------
if page < 1:
wh.CURRENT_EXEC_PAGE = 1
elif page > pg.num_pages:
wh.CURRENT_EXEC_PAGE = pg.num_pages
ms = pg.page(wh.CURRENT_EXEC_PAGE)
self.job_data.data = {}
self.job_data.ok = ms is not None
if not ms:
self.job_data.ok = False
return self.job_data
data = {}
i = 0
for j in ms:
name = j.alg_name
params = {}
params['algorithm'] = {'name': j.alg_name, 'params': {}}
try:
status = j.request.pipe_state.state
date = j.request.pipe_state.started_time
except AttributeError:
status = j.request.pipeline_state.state
date = j.request.pipeline_state.started_time
error = j.request.pipeline_state.exception
pos = error.rfind("Exception: ")
error = error[pos + 11:]
ainfo_ms = algorithm_info_model.objects.filter(
name=j.alg_name)[0]
if ainfo_ms:
qparams = literal_eval(j.params)
mng = algorithm_manager()
ainfo = mng.get_info(ainfo_ms.definition_file, 'input')
for k in ainfo:
if 'view_name' in ainfo[k]:
params['algorithm']['params'][
ainfo[k]['view_name']] = str(
qparams['algorithm']['params'][ainfo[k]
['name']])
else:
params['algorithm']['params'][
ainfo[k]['name']] = str(
qparams['algorithm']['params'][ainfo[k]
['name']])
params = str(params)
else:
params = j.params
#log.info(error)
if not error or error == "":
error = "OK"
if j.is_aborted:
error = "Aborted"
data[i] = (name, status, date, error, j.pk, params)
i += 1
self.job_data.ok = (pg.num_pages, wh.CURRENT_EXEC_PAGE, \
wh.CURRENT_EXEC_PAGE + 1, wh.CURRENT_EXEC_PAGE - 1, alldata, wh.SORTING_EXEC_BY)
self.job_data.data = data
return self.job_data
# OLD
sorted_index = sorted(data, key=lambda x: data[x][1], reverse=True)
i = 0
for ind in sorted_index:
self.job_data.data[i] = data[ind]
i += 1
return self.job_data