def start(self, data): """This method runs the algorithm job. This method will start an algorithm asynchronously. The data parameter must have the key 'algorithm_id' containing the algorithm id to be started. The method will retrieve an algorithm_info_model with the given id and it will create an algorithm_model and save it. By saving the model a algorithm_task will start asynchronously. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True if the algorithm has started, False otherwise. @see algorithm_info_model @link avi.models.algorithm_info_model @see algorithm_model @link avi.models.algorithm_model @see algorithm_task @link avi.task.algorithm_task """ log = logger().get_log("algorithm_task") log.info("job %s", data) if "algorithm_id" in data: mng = algorithm_manager() if not data.get('algorithm_id'): self.job_data.ok = False return self.job_data if len(data['algorithm_id']) < 1: self.job_data.ok = False return self.job_data alg = algorithm_info_model.objects.get(pk=data['algorithm_id'][0]) result = mng.get_algorithm_data(data['algorithm_id'][0], alg.name, alg.definition_file, data) m = algorithm_model(alg_name=alg.name, params=result, results=result) m.save() self.job_data.data = m self.job_data.ok = True return self.job_data # OLD m = algorithm_model(alg_name=data['algorithm']['name'], params=data, results=data) m.save() #log.info("model %s", str(m)) self.job_data.data = m self.job_data.ok = True return self.job_data
def test_init(self): test_log("Testing the initialization method...",self) test_log("Changing the warehouse paths",self) wh = warehouse().get() wh.ALGORITHM_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)) ,'algorithms', 'installed') wh.UPLOADED_ALGORITHM_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)) ,'algorithms', 'uploaded') test_log("Testing the initialization with an empty database",self) mng = algorithm_manager() mng.init() alg = algorithm_info_model.objects.filter(name='installed', algorithm_type='installed') self.assertEqual(len(alg),1) alg = algorithm_info_model.objects.filter(name='temp', algorithm_type='temporal') self.assertFalse(alg) alg = algorithm_info_model.objects.filter(name='uploaded', algorithm_type='uploaded') self.assertEqual(len(alg),1)
def start(self, data): """This method is deprecated""" log = logger().get_log('views') log.info(data) from avi.core.algorithm.algorithm_manager import algorithm_manager res = algorithm_manager().get_algorithm(data) self.job_data.data = {} self.job_data.ok = True return self.job_data
def test_has_param_type(self): test_log("Testing the get_info method...",self) path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'algorithms') mng = algorithm_manager() test_log("Testing an OK file",self) ret = mng.has_param_type(path+"/test.json",'float') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'integer') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'long') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'complex') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'string') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'gaia_table') self.assertTrue(ret) ret = mng.has_param_type(path+"/test.json",'hsa_table') self.assertTrue(ret) test_log("Testing non valid files",self) ret = mng.has_param_type(path+"/test.py",'float') self.assertIsNone(ret) ret = mng.has_param_type(path+"/non_existing_file.json",'float') self.assertIsNone(ret) ret = mng.has_param_type(path,'float') self.assertIsNone(ret) ret = mng.has_param_type("/non/existing/path/test.json",'float') self.assertIsNone(ret) test_log("Testing non valied types",self) ret = mng.has_param_type(path+"/test.json",'non-existiny-type') self.assertFalse(ret) ret = mng.has_param_type(path+"/test.json",None) self.assertFalse(ret)
def get_algorithm_list(self): """Returns the list of algorithms Args: self: The object pointer Returns: A dictionary with the information of all algorithms """ from avi.core.algorithm.algorithm_manager import algorithm_manager return algorithm_manager().get_algorithm_list()
def get_algorithm(self, data): """Returns the algorithm information Args: self: The object pointer data: A JSON file with the information Returns: A dictionary with the algorithm information """ from avi.core.algorithm.algorithm_manager import algorithm_manager return algorithm_manager().get_algorithm_info(data)
def test_get_info(self): test_log("Testing the get_info method...",self) path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'algorithms') mng = algorithm_manager() test_log("Testing an OK file",self) ret = mng.get_info(path+"/test.json",'name') self.assertEqual(self._def['algorithm']['name'], ret) self.assertEqual(str(self._def['algorithm']['name']), str(ret)) test_log("Testing non valid files",self) ret = mng.get_info(path+"/test.py",'name') self.assertIsNone(ret) self.assertNotEqual(self._def['algorithm']['name'], ret) self.assertNotEqual(str(self._def['algorithm']['name']), str(ret)) ret = mng.get_info(path+"/non_existing_file.json",'name') self.assertIsNone(ret) self.assertNotEqual(self._def['algorithm']['name'], ret) self.assertNotEqual(str(self._def['algorithm']['name']), str(ret)) ret = mng.get_info(path,'name') self.assertIsNone(ret) self.assertNotEqual(self._def['algorithm']['name'], ret) self.assertNotEqual(str(self._def['algorithm']['name']), str(ret)) ret = mng.get_info("/non/existing/path/test.json",'name') self.assertIsNone(ret) self.assertNotEqual(self._def['algorithm']['name'], ret) self.assertNotEqual(str(self._def['algorithm']['name']), str(ret)) test_log("Testing non valied keys",self) ret = mng.get_info(path+"/test.json",'non-existiny-key') self.assertIsNone(ret) ret = mng.get_info(path+"/test.json",None) self.assertIsNone(ret)
def _create_algorithm_info(self): mng = algorithm_manager() alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)) ,'algorithms', 'installed') mng.update_database(alg_dir, 'installed') alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)) ,'algorithms', 'temp') mng.update_database(alg_dir,'temporal') alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)) ,'algorithms', 'uploaded') mng.update_database(alg_dir,'uploaded')
def test_get_algorithm_data(self): test_log("Testing the get_info method...",self) alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)) ,'algorithms') #algorithm_info_model.objects.create(name="test", # source_file=alg_dir + "/test.py", # name_view="Test View", # definition_file=alg_dir + # "/test.json", # algorithm_type="installed") mng = algorithm_manager() test_log("Testing an OK file and OK post",self) post = self._post_exec_algorithm() ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post) self.assertEqual(sorted(ret), sorted(self._val)) self.assertEqual(str(sorted(ret)), str(sorted(self._val))) test_log("Testing non valid files",self) ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.py",post) self.assertIsNone(ret) ret = mng.get_algorithm_data('1','test', alg_dir+"/non_existing_file.json",post) self.assertIsNone(ret) ret = mng.get_algorithm_data('1','test', alg_dir,post) self.assertIsNone(ret) ret = mng.get_algorithm_data('1','test',"/non/existing/path/test.json", post) self.assertIsNone(ret) test_log("Testing non valid POST",self) post = None ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post) self.assertIsNone(ret) post = {} ret = mng.get_algorithm_data("1", 'test', alg_dir + "/test.json", post) self.assertIsNone(ret)
def test_get_algorithm(self): test_log("Testing the get_algorithm method...",self) path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'algorithms') mng = algorithm_manager() test_log("Testing an OK file",self) ret = mng.get_algorithm(path+"/test.json") self.assertEqual(self._def, ret) self.assertEqual(str(self._def), str(ret)) test_log("Testing non valid files",self) ret = mng.get_algorithm(path+"/test.py") self.assertIsNone(ret) self.assertNotEqual(self._def, ret) self.assertNotEqual(str(self._def), str(ret)) ret = mng.get_algorithm(path+"/non_existing_file.json") self.assertIsNone(ret) self.assertNotEqual(self._def, ret) self.assertNotEqual(str(self._def), str(ret)) ret = mng.get_algorithm(path) self.assertIsNone(ret) self.assertNotEqual(self._def, ret) self.assertNotEqual(str(self._def), str(ret)) ret = mng.get_algorithm("/non/existing/path/test.json") self.assertIsNone(ret) self.assertNotEqual(self._def, ret) self.assertNotEqual(str(self._def), str(ret))
def start(self, data): """This method retrieves the algorithm information. The data parameter must have the key 'id' containing the algorithm id which info will be retrieved. The method will retrieve the algorithm_info_model with the given id and it will will extract the parameters information from the definition file provided by the algorithm_info_model. If the algorithm has gaia, herschel or results files in its input, this method will also retrieve the names of thouse files from the resource_model. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True if the algorithm exists, False otherwise. @see algorithm_info_model @link avi.models.algorithm_info_model @see resource_model @link avi.models.resource_model """ log = logger().get_log('algorithm_manager') log.info(data) am = algorithm_info_model.objects.get(pk=data['id']) if not am: self.log_data.ok = False return self.job_data alg_info = algorithm_manager().get_algorithm(am.definition_file) inp = alg_info['algorithm']['input'] #inp = {} #inp = sorted(inp.items()) #inp = sorted(inp, key = lambda x: (x.get('group') is None, x)) self.job_data.data = dict() self.job_data.data['algorithm'] = alg_info['algorithm'] inp = [] for _, i in alg_info['algorithm']['input'].items(): log.info(i) inp.append(i) inp.sort(key=lambda x: x['position'], reverse=False) # self.job_data.data['algorithm_input'] = inp self.job_data.data.update({'algorithm_input':inp}) ms = resource_model.objects.all() has_gaia = algorithm_manager().has_param_type(am.definition_file, 'gaia_table') if has_gaia: # self.job_data.data['gaia'] = [] self.job_data.data.update({'gaia':[]}) has_hsa = algorithm_manager().has_param_type(am.definition_file, 'hsa_table') if has_hsa: # self.job_data.data['hsa'] = [] self.job_data.data.update({'hsa':[]}) log.info(am.definition_file) has_results = algorithm_manager().has_param_type(am.definition_file, 'results_data') if has_results: # self.job_data.data['res'] = [] self.job_data.data.update({'res':[]}) has_user = algorithm_manager().has_param_type(am.definition_file, 'user_data') if has_user: # self.job_data.data['user'] = [] self.job_data.data.update({'user':[]}) for i in ms: log.info("file %s - %s", i.name, i.file_type) if has_gaia and i.file_type == 'gaia': self.job_data.data['gaia'].extend([i.name]) if has_hsa and i.file_type == 'hsa': self.job_data.data['hsa'].extend([i.name]) if has_results and i.file_type == 'result': self.job_data.data['res'].extend([i.name]) if has_user and i.file_type == 'user': self.job_data.data['user'].extend([i.name]) #self.job_data.data = alg_info self.job_data.ok = alg_info is not None return self.job_data
def start(self, data): """This method runs the get_algorithms job. If the algorithms are not loaded it will load them. Then it will retrieve the all algorithm_info_models and return the data from them. Args: self: The object pointer. data: A dictionary containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True if there are algorithms retrieved, False otherwise. @see algorithm_info_model @link avi.models.algorithm_info_model """ log = logger().get_log('algorithm_manager') wh_f = wh_frontend_config().get() if not wh().get().ALGORITHMS_LOADED: from avi.core.algorithm.algorithm_manager import algorithm_manager algorithm_manager().init() wh().get().ALGORITHMS_LOADED = True from avi.models import algorithm_info_model, algorithm_group_model all_ms = algorithm_info_model.objects.all().order_by( 'name_view', 'name', 'pk') #sall_ms = sorted(all_ms, key = lambda x:(x.name_view is None, x)) self.job_data.data = {} self.job_data.ok = all_ms is not None if not all_ms: return self.job_data pg = Paginator(all_ms, wh_f.MAX_ALG_PER_PAGE) page = wh_f.CURRENT_ALG_PAGE if page < 1: wh_f.CURRENT_ALG_PAGE = 1 elif page > pg.num_pages: wh_f.CURRENT_ALG_PAGE = pg.num_pages ms = pg.page(wh_f.CURRENT_ALG_PAGE) all_ms_g = algorithm_group_model.objects.all().order_by( 'name_view', 'name', 'pk') pg_g = Paginator(all_ms_g, wh_f.MAX_ALG_PER_PAGE) page = wh_f.CURRENT_ALG_PAGE if page < 1: wh_f.CURRENT_ALG_PAGE = 1 elif page > pg.num_pages: wh_f.CURRENT_ALG_PAGE = pg.num_pages ms_g = pg_g.page(wh_f.CURRENT_ALG_PAGE) data = [] for g in ms_g: data.append({"group": g, "algorithms": []}) data.sort(key=lambda x: x["group"].position, reverse=False) for j in ms: for g in data: if j.algorithm_group == g["group"].name: g["algorithms"].append( (j.pk, j.name, j.name_view, j.algorithm_type, j.algorithm_group, j.position)) for g in data: g["algorithms"].sort(key=lambda x: x[4], reverse=False) for g in data: log.info(g["group"].name) log.info(g["group"].position) for a in g["algorithms"]: log.info(a[4]) log.info(a[1]) # OLD # data = {} # i = 0 # for j in ms: # data[i] = (j.pk, # j.name, # j.name_view, # j.algorithm_type) # i += 1 res = {} res["algorithms"] = data res["max_pages"] = pg.num_pages res["current_page"] = wh_f.CURRENT_ALG_PAGE res["next_page"] = wh_f.CURRENT_ALG_PAGE + 1 res["prev_page"] = wh_f.CURRENT_ALG_PAGE - 1 self.job_data.data = res return self.job_data
def test_job_abort_start_ok(self): input_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'input') alg_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'algorithms') test_log("Input path: %s" % (input_dir), self) test_log("Algorithms path: %s" % (alg_dir), self) algorithm_info_model.objects.create(name="test", source_file=alg_dir + "/test.py", name_view="Test View", definition_file=alg_dir + "/test.json", algorithm_type="installed") #test_info("Starting...",self) pk = 1 try: alg_info = algorithm_info_model.objects.get(pk=pk) except algorithm_info_model.DoesNotExist: #test_info("[FAILED]",self) self.assertTrue(False) mng = algorithm_manager() self.assertIsNotNone(mng) post = self._post_exec_algorithm(pk, alg_info.name, p5="WAIT_ABORT") alg_input = mng.get_algorithm_data(str(pk), alg_info.name, alg_info.definition_file, post) alg_exec = algorithm_model.objects.create(alg_name=alg_info.name, params=alg_input, results={}) alg_exec.save() while alg_exec.request.pipeline_state.state != 'PENDING': #'STARTED': try: alg_exec = algorithm_model.objects.get(pk=pk) except algorithm_model.DoesNotExist: #test_info("[FAILED]", self) self.assertTrue(False) #test_info(alg_exec.request.pipeline_state.state) time.sleep(1) status = alg_exec.request.pipeline_state.state self.assertNotEqual(status, 'FAILURE') self.assertNotEqual(status, 'SUCCESS') self.assertFalse(alg_exec.is_aborted) from avi.core.pipeline.job_abort import abort res = abort().start(self._post_abort('algorithm', pk)) self.assertTrue(res.ok) try: alg_exec = algorithm_model.objects.get(pk=pk) except algorithm_model.DoesNotExist: #test_info("[FAILED]", self) self.assertTrue(False) #test_info(str(res.data.params)) #test_info(str(alg_exec.params)) #while alg_exec.request.pipeline_state.state != 'FAILURE': #test_info(alg_exec.request.pipeline_state.state) # time.sleep(1) status = alg_exec.request.pipeline_state.state self.assertEqual(status, 'FAILURE') self.assertTrue(alg_exec.is_aborted)
def start(self, data): """This method runs the get_pipeline_status job. This method will retrieve the algorithm_models and it will sort them by the current sorting method provided by the wh_frontend_config warehouse. Then it will paginate the results with current page retrieved from the wh_frontend_config warehouse. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute provides the pages information. @see algorithm_model @link avi.models.algorithm_model @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ log = logger().get_log("get_pipeline_status") wh = wh_frontend_config().get() sorting_wh = wh.SORTING_EXEC_BY order_by = 'request__pipeline_state__started_time' if sorting_wh == 'name': order_by = 'alg_name' elif sorting_wh == '-name': order_by = '-alg_name' elif sorting_wh == '-date': order_by = '-request__pipeline_state__started_time' elif sorting_wh == 'status': order_by = 'request__pipeline_state__state' elif sorting_wh == '-status': order_by = '-request__pipeline_state__state' all_ms = algorithm_model.objects.all().order_by(order_by, 'pk') self.job_data.data = {} self.job_data.ok = all_ms is not None if not all_ms: self.job_data.ok = False return self.job_data pg = Paginator(all_ms, wh.MAX_EXEC_PER_PAGE) page = wh.CURRENT_EXEC_PAGE #------------------ #----all data algorithms alldata = {} k = 0 for h in pg.object_list: try: status = h.request.pipe_state.state date = h.request.pipe_state.started_time except AttributeError: status = h.request.pipeline_state.state date = h.request.pipeline_state.started_time #status = h.request.pipeline_state.state #date = h.request.pipeline_state.started_time params = {} params['algorithm'] = {'name': h.alg_name, 'params': {}} ainfo_ms = algorithm_info_model.objects.filter(name=h.alg_name)[0] if ainfo_ms: qparams = literal_eval(h.params) mng = algorithm_manager() ainfo = mng.get_info(ainfo_ms.definition_file, 'input') for l in ainfo: if 'view_name' in ainfo[l]: params['algorithm']['params'][ainfo[l][ 'view_name']] = qparams['algorithm']['params'][ ainfo[l]['name']] else: params['algorithm']['params'][ ainfo[l]['name']] = qparams['algorithm']['params'][ ainfo[l]['name']] params = str(params) else: params = h.params alldata[k] = (h.alg_name, h.pk, params, date, status) log.info(params) k += 1 #------------------ if page < 1: wh.CURRENT_EXEC_PAGE = 1 elif page > pg.num_pages: wh.CURRENT_EXEC_PAGE = pg.num_pages ms = pg.page(wh.CURRENT_EXEC_PAGE) self.job_data.data = {} self.job_data.ok = ms is not None if not ms: self.job_data.ok = False return self.job_data data = {} i = 0 for j in ms: name = j.alg_name params = {} params['algorithm'] = {'name': j.alg_name, 'params': {}} try: status = j.request.pipe_state.state date = j.request.pipe_state.started_time except AttributeError: status = j.request.pipeline_state.state date = j.request.pipeline_state.started_time error = j.request.pipeline_state.exception pos = error.rfind("Exception: ") error = error[pos + 11:] ainfo_ms = algorithm_info_model.objects.filter( name=j.alg_name)[0] if ainfo_ms: qparams = literal_eval(j.params) mng = algorithm_manager() ainfo = mng.get_info(ainfo_ms.definition_file, 'input') for k in ainfo: if 'view_name' in ainfo[k]: params['algorithm']['params'][ ainfo[k]['view_name']] = str( qparams['algorithm']['params'][ainfo[k] ['name']]) else: params['algorithm']['params'][ ainfo[k]['name']] = str( qparams['algorithm']['params'][ainfo[k] ['name']]) params = str(params) else: params = j.params #log.info(error) if not error or error == "": error = "OK" if j.is_aborted: error = "Aborted" data[i] = (name, status, date, error, j.pk, params) i += 1 self.job_data.ok = (pg.num_pages, wh.CURRENT_EXEC_PAGE, \ wh.CURRENT_EXEC_PAGE + 1, wh.CURRENT_EXEC_PAGE - 1, alldata, wh.SORTING_EXEC_BY) self.job_data.data = data return self.job_data # OLD sorted_index = sorted(data, key=lambda x: data[x][1], reverse=True) i = 0 for ind in sorted_index: self.job_data.data[i] = data[ind] i += 1 return self.job_data