def __init__(self): """Constructor Initializes the log, the warehouses, the configuration and the interface_manager Args: self: The object pointer """ ipath = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'config') #wh_global_config().get().INPUT_PATH self.str_log_config_file = os.path.join(ipath, 'log_config.xml') self.str_config_file = os.path.join(ipath, 'config.xml') self.str_global_config_file = os.path.join(ipath, 'global_config.xml') from avi.utils.config_manager import logger_configuration log_cfg = logger_configuration() log_cfg.load(self.str_log_config_file) self.log = logger().get_log(self.str_log_header) self.log.info("Initialization of RISEA...") self.log.info("RISEA ptr : %s",self.get_ptr()) # loading cfg from avi.utils.config_manager import configuration_manager self.cfg = configuration_manager() if not self.cfg.load(self.str_config_file): self.log.error("Failed to load the configuration %s", self.str_config_file) return # Initialization of the warehouses cfg = configuration_manager() if not cfg.load(self.str_global_config_file): self.log.error("Failed to load the configuration %s", self.str_global_config_file) return wh_global_config().get().load(cfg.get("global")) wh_frontend_config().get().load(cfg.get("frontend")) wh_frontend_config().get().\ CURRENT_PATH = os.path.\ normpath("".join(wh_global_config().get().RESOURCES_PATH)) wh_frontend_config().get().\ HOME_PATH = os.path.\ normpath("".join(wh_global_config().get().RESOURCES_PATH)) self.log.info("current id : %s\n resources id : %s", id(wh_frontend_config().get().CURRENT_PATH), id(wh_global_config().get().RESOURCES_PATH)) # Initialization of the interface manager from .interface.interface_manager import interface_manager self.interface_manager = interface_manager() self.interface_manager.init(self.cfg) # Initialization of the resources from avi.utils.resources_manager import resources_manager resources_manager().init()
def rename_directory(self, name, new_name): """Deprecated""" folder = \ wh_frontend_config().get().CURRENT_PATH + "/" + name new_folder = \ wh_frontend_config().get().CURRENT_PATH + "/" + new_name if os.path.isdir(folder): if os.path.isdir(new_folder): self.log.info('The directory already exists.') else: os.rename(folder, new_folder) else: self.log.info('The directory does not exists.')
def rename_file(self, name, new_name): """Deprecated""" old_file = \ wh_frontend_config().get().CURRENT_PATH + "/" + name new_file = \ wh_frontend_config().get().CURRENT_PATH + "/" + new_name if os.path.isfile(old_file): if os.path.isfile(new_file): self.log.info('The file already exists.') else: os.rename(old_file, new_file) else: self.log.info('The file does not exists.')
def start(self, data): """This method runs the relaunch_algorithm job. This method will relaunch_algorithm the asynchronous job provided in the data parameter. The data parameter must have the key 'algorithm' containing all the data of the algorithm to be relaunched. It will first parse the data parameter to a dictionary and save it into 'qparams' variable, then get the algorithm model by the name in the 'qparams' variable and save the 'pk' of the algorithm into 'qparams. If the algorithm input parameters contains 'Gaia' then will set the gaia file, if contains 'HSA' then will set the hsa file. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The algorithm parameters to be relaunched. """ wh = wh_frontend_config().get() gwh = wh_global_config().get() log = logger().get_log("relaunch") log.info("inside relaunch_algorithm job") # data is a string type, transform to dictionary qparams = literal_eval(data['pk']) # get the model of the algorithm by the name in qparams m = algorithm_info_model.objects.get(name=qparams['algorithm']['name']) # add the pk of the algorithm qparams['algorithm']['pk'] = m.pk # get the keys of the algorithm parameters keys = qparams['algorithm']['params'].keys() # check if Gaia or HSA is one of the parameters to delete the path of the files if "Gaia" in keys: path_to_eliminate = gwh.SOURCES_PATH path_to_eliminate = str(path_to_eliminate) + "/gaia/" qparams['algorithm']['params']['Gaia'] = qparams['algorithm'][ 'params']['Gaia'].replace(path_to_eliminate, '') log.info(qparams['algorithm']['params']['Gaia']) #Delete path if "HSA" in keys: path_to_eliminate = gwh.SOURCES_PATH path_to_eliminate = str(path_to_eliminate) + "/hsa/" qparams['algorithm']['params']['HSA'] = qparams['algorithm'][ 'params']['HSA'].replace(path_to_eliminate, '') log.info(qparams['algorithm']['params']['HSA']) #Delete path data = {} self.job_data.data = qparams['algorithm'] log.info("params " + str(qparams['algorithm'])) return self.job_data
def start(self, data): """This method runs the sorting_by job. This methods changes the current sorting method storaged in the wh_frontend_config warehouse. The data parameter must have the key 'page' containing the name of the web page whose sorting method will change. It must have also the key 'sort_by' with the sorting method to be changed with. Here we just change the current sorting method in the warehouse, without checking if it is valid or not. Those error controls will be made in other jobs. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True always. @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ wh = wh_frontend_config().get() if data['page'] == "pipeline_status": if self.are_equal(wh.SORTING_EXEC_BY, data['sort_by']): wh.SORTING_EXEC_BY = self.invert(wh.SORTING_EXEC_BY) else: wh.SORTING_EXEC_BY = data['sort_by'] elif data['page'] == "query_status": if self.are_equal(wh.SORTING_QUERY_BY, data['sort_by']): wh.SORTING_QUERY_BY = self.invert(wh.SORTING_QUERY_BY) else: wh.SORTING_QUERY_BY = data['sort_by'] elif data['page'] == "algorithm": if self.are_equal(wh.SORTING_ALG_BY, data['sort_by']): wh.SORTING_ALG_BY = self.invert(wh.SORTING_ALG_BY) else: wh.SORTING_ALG_BY = data['sort_by'] elif data['page'] == "resources": if self.are_equal(wh.SORTING_RESOURCES_BY, data['sort_by']): wh.SORTING_RESOURCES_BY = self.invert(wh.SORTING_RESOURCES_BY) else: wh.SORTING_RESOURCES_BY = data['sort_by'] elif data['page'] == "results": if self.are_equal(wh.SORTING_RESULTS_BY, data['sort_by']): wh.SORTING_RESULTS_BY = self.invert(wh.SORTING_RESULTS_BY) else: wh.SORTING_RESULTS_BY = data['sort_by'] self.job_data.data = {} self.job_data.ok = True return self.job_data
def directory_down(self, path, down_dir): """Moves the current warehouse path to the given path specified directory This method changes the CURRENT_PATH stored in the wh_frontend_config warehouse to the given path spacified directory Args: self: The object pointer path: The path from which the down_dir directory must be down_dir: The new current path Returns: The new current path if everything went correctly See: wh_frontend_config: avi.warehouse.wh_frontend_config """ wh_frontend_config().get().CURRENT_PATH = \ self.move_absolute_directory(os.path.join(path, down_dir)) return wh_frontend_config().get().CURRENT_PATH
def directory_up(self): """Moves the warehouse current path to the parent directory Args: self: The object pointer Returns: The warehouse current path """ from avi.utils.resources_manager import resources_manager return resources_manager() \ .directory_up(wh_frontend_config().get().CURRENT_PATH)
def create_directory(self, directory): """Deprecated""" import re, errno if re.match("^[a-zA-Z0-9_ ]*$", directory): path_full = \ wh_frontend_config().get().CURRENT_PATH + "/" + directory try: os.makedirs(path_full) except OSError as e: if e.errno != errno.EEXIST: raise else: raise
def move_default_directory(self): """Moves the warehouse current path to the home directory Args: self: The object pointer Returns: The warehouse current path """ from avi.utils.resources_manager import resources_manager return resources_manager() \ .move_absolute_directory("".join(wh_frontend_config().\ get().HOME_PATH))
def directory_up(self, path): """Moves the current warehouse path to the parent of the given path This method changes the CURRENT_PATH stored in the wh_frontend_config warehouse to the parent of the given path Args: self: The object pointer path: The path which parent will be the new current path Returns: The new current path if everything went correctly, None otherwise See: wh_frontend_config: avi.warehouse.wh_frontend_config """ if path == wh_frontend_config().get().HOME_PATH: self.log.info('The directory does not exist.') else: wh_frontend_config() \ .get().CURRENT_PATH = \ self.move_absolute_directory(os.path.split(path)[0]) return wh_frontend_config().get().CURRENT_PATH
def directory_down(self, folder_local): """Moves the warehouse current path to the given new location Args: self: The object pointer folder_local: The directory to enter Returns: The warehouse current path """ from avi.utils.resources_manager import resources_manager return resources_manager() \ .directory_down(wh_frontend_config().get().\ CURRENT_PATH, folder_local)
def start(self, data): """This method runs the change page job. This methods changes the current page storaged in the wh_frontend_config warehouse. The data parameter must have the key 'page' containing the name of the web page whose page number will change. It must have also the key 'number' with the number of the page. Here we just change the current page in the warehouse, without checking if it is valid or not. Those error controls will be made in other jobs. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True always. @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ #log = logger().get_log('get_queries_status') wh = wh_frontend_config().get() if data['page'] == "pipeline_status": wh.CURRENT_EXEC_PAGE = int(data['number']) if data['page'] == "query_status": wh.CURRENT_QUERY_PAGE = int(data['number']) if data['page'] == "algorithm": wh.CURRENT_ALG_PAGE = int(data['number']) if data['page'] == "resources": wh.CURRENT_RESOURCES_PAGE = int(data['number']) if data['page'] == "results": wh.CURRENT_RESULTS_PAGE = int(data['number']) self.job_data.data = {} self.job_data.ok = True return self.job_data
def delete_directory(self, directory): """Deprecated""" import shutil folder = \ wh_frontend_config().get().CURRENT_PATH + "/" + directory if os.path.isdir(folder): for the_file in os.listdir(folder): file_path = os.path.join(folder, the_file) if os.path.isfile(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) else: self.log.info('The directory does not exist.') if os.path.isdir(folder): os.rmdir(folder) else: self.log.info('The directory does not exist.') else: self.log.info('The directory does not exist.')
def move_absolute_directory(self, path): """Moves the current warehouse path to the given path This method changes the CURRENT_PATH stored in the wh_frontend_config warehouse to the given path Args: self: The object pointer path: The new current path Returns: The new current path if everything went correctly, False otherwise See: wh_frontend_config: avi.warehouse.wh_frontend_config """ if self.dir_exists(path): new_directory = wh_frontend_config().get().CURRENT_PATH = path return new_directory else: self.log.info('The directory does not exist.') return False
def delete_file(self, file_name): """Deletes the given file Args: self: The object pointer file_name: the file to be deleted Raises: Exception: if the an unknow error happens """ # file_to_delete = \ # wh_frontend_config().get().CURRENT_PATH + "/" + file_name path_to_delete = wh_frontend_config().get().CURRENT_PATH self.log.info(path_to_delete) self.log.info(file_name) self.log.info(file_name.startswith('gaia')) if file_name.startswith('gaia'): if 'gaia' not in path_to_delete: path_to_delete = "/data/output/sources/gaia" elif file_name.startswith('hsa'): if 'hsa' not in path_to_delete: path_to_delete = "/data/output/sources/hsa" elif file_name.startswith('res'): if 'results' not in path_to_delete: path_to_delete = "/data/output/results" elif file_name.startswith('user'): if 'user' not in path_to_delete: path_to_delete = "/data/output/user" try: # os.remove(file_to_delete) file_manager().remove_file(file_name, path_to_delete) # this would be "except OSError, e:" before Python 2.6 except OSError as e: # errno.ENOENT = no such file or directory if e.errno != errno.ENOENT: raise # re-raise exception if a different error occurred
def start(self, data): """This method runs the launch job. This method will launch the asynchronous job provided in the data parameter. The data parameter must have the key 'id' containing the primary key of the query and the mission to be launched and the key mission unseted. It will first split the key 'id' by '-' and save the primary key into data 'id' and the mission into data 'mission'. Then it will get all the algorithms and take the valid algorithms for the query by comparing the data 'mission' and the inputs of the algorithms. Finally it will set all the parameters to be returned into the 'ret' variable. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The the mission. The algorithms. The files of the query """ wh = wh_frontend_config().get() gwh = wh_global_config().get() log = logger().get_log("launch") log.info("inside launch job") ret={} cont = 0 ids = [] data['mission'] = data['id'].split('-')[1] data['id'] = data['id'].split('-')[0] ret['mission'] = data['mission'] ret['algorithms'] = {} pk={} #Take valid alorithms for the query m = get_algorithms.start(self, None) #for i in range(len(m.data['algorithms'])): # ids.append(m.data['algorithms'][i][0]) log.info(m.data) for group in m.data['algorithms']: for i in group['algorithms']: ids.append(i[0]) for j in ids: pk['id'] = j model = get_algorithm_info.start(self, pk) for a in list(model.data['algorithm']['input'].keys()): #if ret['mission'].lower() == model.data['algorithm']['input'][a]['view_name'].lower(): if ret['mission'].lower() in model.data['algorithm']['input'][a]['type'].lower(): ret['algorithms'][cont] = {} ret['algorithms'][cont]['pk'] = pk['id'] ret['algorithms'][cont]['view_name'] = model.data['algorithm']['view_name'] log.info(cont) cont = cont + 1 break #Set all the return data query_info = get_query_info.start(self,data) query_info_data = dict(query_info.data) if 'files' in query_info_data: ret['files'] = query_info_data['files'] else: ret['files'] = 'No data found' log.info("return of job_launch: "+str(ret)) self.job_data.data = ret return self.job_data
def start(self, data): """This method runs the get_queries_status job. This method will retrieve the gaia_query_models and the herschel_query_models and it will sort them by the current sorting method provided by the wh_frontend_config warehouse. Then it will paginate the results with current page retrieved from the wh_frontend_config warehouse. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute provides the pages information. @see gaia_query_model @link avi.models.gaia_query_model @see herschel_query_model @link avi.models.herschel_query_model @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ log = logger().get_log("get_queries_status") current_started = wh_common().get().queries_started count_started = 0 wh = wh_frontend_config().get() sorting_wh = wh.SORTING_QUERY_BY order_by = 'request__pipeline_state__started_time' order_lambda = lambda query: (query.request.pipeline_state. started_time, query) if sorting_wh == 'name': order_by = 'name' order_lambda = lambda query: ("%s %s %s" % (query.archive, query. name, query.pk), query) elif sorting_wh == '-name': order_by = '-name' order_lambda = lambda query: ("%s %s %s" % (query.archive, query. name, query.pk), query) elif sorting_wh == '-date': order_by = '-request__pipeline_state__started_time' order_lambda = lambda query: (query.request.pipeline_state. started_time, query) elif sorting_wh == 'status': order_by = 'request__pipeline_state__state' order_lambda = lambda query: (query.request.pipeline_state.state, query) elif sorting_wh == '-status': order_by = '-request__pipeline_state__state' order_lambda = lambda query: (query.request.pipeline_state.state, query) all_gaia = gaia_query_model.objects.all().order_by(order_by, 'pk') all_hsa = herschel_query_model.objects.all().order_by(order_by, 'pk') all_sim = sim_query_model.objects.all().order_by(order_by, 'pk') len_all = len(str(len(all_gaia) + len(all_hsa) + len(all_sim))) order_lambda = lambda query: (query.request.pipeline_state. started_time, query) if sorting_wh == 'name': order_lambda = lambda query: ("%s %s %s" % ( query.archive, query.name, str(query.pk).zfill(len_all)), query ) elif sorting_wh == '-name': order_lambda = lambda query: ("%s %s %s" % ( query.archive, query.name, str(query.pk).zfill(len_all)), query ) log.info("models retrieved") self.job_data.data = dict([(0, False), (1, {})]) self.job_data.ok = False #pgg = Paginator(all_gaia, wh.MAX_QUERY_PER_PAGE) #pgh = Paginator(all_hsa, wh.MAX_QUERY_PER_PAGE) #page_gaia = wh.CURRENT_QUERY_PAGE #if page_gaia < 1: # wh.CURRENT_QUERY_PAGE = 1 #elif page_gaia > pgg.num_pages: # wh.CURRENT_QUERY_PAGE = pgg.num_pages #ms_gaia = pgg.page(wh.CURRENT_QUERY_PAGE) #if wh.CURRENT_QUERY_PAGE > pgh.num_pages: # wh.CURRENT_QUERY_PAGE = pgh.num_pages #ms_hsa = pgh.page(wh.CURRENT_QUERY_PAGE) reverse = False if order_by[0] == '-': reverse = True order_by.replace('-', '', 1) call = chain(all_gaia, all_hsa, all_sim) #for call_ in call: # call_.name = "query %s"%(call_.name) # if isinstance(call_, gaia_query_model): # call_.name = "Gaia %s"%(call_.name) # elif isinstance(call_, herschel_query_model): # call_.name = "HSA %s"%(call_.name) # elif isinstance(call_, sim_query_model): # call_.name = "SIM %s"%(call_.name) #try: all_ms = sorted( call, #key=attrgetter("name"), key=order_lambda, reverse=reverse) #for am in all_ms: # log.info("%s %s %s", am.name, am.request.pipeline_state.started_time, am.request.pipeline_state.state) #except TypeError: # log.info("whaaat") # all_ms = sorted(call , #key=attrgetter("name"), # key = lambda query: (query.name is None, query), # reverse=True) log.info("models chained and sorted") pg = Paginator(all_ms, wh.MAX_QUERY_PER_PAGE) page = wh.CURRENT_QUERY_PAGE #log.info(page) #------------------ #---all data queries allqueries = {} k = 0 for h in pg.object_list: try: status = h.request.pipe_state.state date = h.request.pipe_state.started_time except AttributeError: status = h.request.pipeline_state.state date = h.request.pipeline_state.started_time #status = h.request.pipeline_state.state #date = h.request.pipeline_state.started_time name = "query %s" % (h.name) if isinstance(h, gaia_query_model): name = "Gaia %s" % (h.name) elif isinstance(h, herschel_query_model): name = "HSA %s" % (h.name) elif isinstance(h, sim_query_model): name = "SIM %s" % (h.name) allqueries[k] = (name, status, date, h.pk, h.archive) k += 1 #------------------ log.info("paginator done") if page < 1: wh.CURRENT_QUERY_PAGE = 1 elif page > pg.num_pages: wh.CURRENT_QUERY_PAGE = pg.num_pages all_ms = pg.page(wh.CURRENT_QUERY_PAGE) if not all_ms: return self.job_data data = {} #------query info ----- #query_id_mission = {} #query_info = {} #------query info ----- i = 0 for q in all_ms: try: status = q.request.pipe_state.state date = q.request.pipe_state.started_time except AttributeError: status = q.request.pipeline_state.state date = q.request.pipeline_state.started_time error = q.request.pipeline_state.exception pos = error.rfind("Exception: ") error = error[pos + 11:] #------query info ----- #query_id_mission['mission'] = q.archive #query_id_mission['id'] = q.pk #query_info = get_query_info.start(self,query_id_mission) #log.info("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") #log.info(query_info.data) #log.info("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") #------query info ----- if not error or error == "": error = "OK" if q.is_aborted: error = "Aborted" log.info("status: %s", status) if status == "STARTED": count_started += 1 #data[q.pk] = (q.name, status) #------query info ----- #data[i] = ("query %s"%(q.name), status, date, error, q.pk, q.archive, query_info.data) #------query info ----- name = "query %s" % (q.name) if isinstance(q, gaia_query_model): name = "Gaia %s" % (q.name) elif isinstance(q, herschel_query_model): name = "HSA %s" % (q.name) elif isinstance(q, sim_query_model): name = "SIM %s" % (q.name) data[i] = (name, status, date, error, q.pk, q.archive) log.info("loop") i += 1 log.info("looping through the paginator") wh_common().get().queries_started = count_started #if current_started != 0: log.info("asdasd") log.info(self.job_data.data) self.job_data.data[0] = count_started < current_started #self.job_data.data.append(count_started < current_started) self.job_data.ok = [pg.num_pages, wh.CURRENT_QUERY_PAGE, \ wh.CURRENT_QUERY_PAGE + 1, wh.CURRENT_QUERY_PAGE - 1, allqueries, wh.SORTING_QUERY_BY] try: log.info(self.job_data.data) #log.info(self.job_data.data[1]) log.info(data) aux = data log.info(type(aux)) if isinstance(self.job_data.data, dict): #log.info(type(self.job_data.data[1])) self.job_data.data[ 1] = aux #{0:('sim','success'),1:('sim20','success')} else: self.job_data.data.append(data) #self.job_data.data.append(data) except Exception as e: log.info(e) log.info("job_get_queries_status end") return self.job_data # OLD ms = gaia_query_model.objects.all() data = {} i = 0 for q in ms: log.info("query %s", q.name) try: status = q.request.pipe_state.state date = q.request.pipe_state.started_time except AttributeError: status = q.request.pipeline_state.state date = q.request.pipeline_state.started_time error = q.request.pipeline_state.exception pos = error.rfind("Exception: ") error = error[pos + 11:] if not error or error == "": error = "OK" if q.is_aborted: error = "Aborted" log.info("status: %s", status) if status == "STARTED": count_started += 1 #data[q.pk] = (q.name, status) data[i] = ("Gaia %s" % (q.name), status, date, error, q.pk, "gaia") i += 1 #data.append((q.name, status)) ms = herschel_query_model.objects.all() for q in ms: log.info("query %s", q.name) try: status = q.request.pipe_state.state date = q.request.pipe_state.started_time except AttributeError: status = q.request.pipeline_state.state date = q.request.pipeline_state.started_time error = q.request.pipeline_state.exception pos = error.rfind("Exception: ") error = error[pos + 11:] if not error or error == "": error = "OK" if q.is_aborted: error = "Aborted" log.info("status: %s", status) if status == "STARTED": count_started += 1 data[i] = ("HSA %s" % (q.name), status, date, error, q.pk, "hsa") i += 1 sorted_index = sorted(data, key=lambda x: data[x][1], reverse=True) res = {} i = 0 for ind in sorted_index: res[i] = data[ind] i += 1 self.job_data.data = {} self.job_data.data[1] = res wh_common().get().queries_started = count_started #if current_started != 0: self.job_data.data[0] = count_started < current_started #else: # self.job_data.data[0] = False self.job_data.ok = ms is not None return self.job_data self.job_data.data = {} i = 0 for ind in sorted_index: self.job_data.data[i] = data[ind] i += 1 self.job_data.ok = ms is not None return self.job_data
def start(self, data): """This method runs the get_files job. This method will retrieve all the allowed files and directories to be shown in the user interface. It uses the resources_manager to get the path information and then uses the discard_files to discard the ones that should not be shown. After that it will paginate the results with the current page retrieved from the wh_frontend_config warehouse. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute provides the pages information. @see resources_manager @link avi.utils.resources_manager.resources_manager @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ log = logger().get_log("views") wh = wh_frontend_config().get() gwh = wh_global_config().get() #dirs = data[0] #files = data[1] sorting_wh = wh.SORTING_RESOURCES_BY if sorting_wh[0] == '-': sorting_wh = sorting_wh[1:] order_by = '' all_files = self.discard_files(resources_manager().get_list( wh.CURRENT_PATH)) #---------------------------------------------------------------------------------------------------- #log.info("Current path!!" + str(wh.CURRENT_PATH)) #log.info("all filess: " + str(all_files)) gaia_files = self.discard_files( resources_manager().get_list("/data/output/sources/gaia")) #log.info("gaia filess: " + str(gaia_files)) gaia = resources_manager().get_info(gaia_files, "/data/output/sources/gaia") #log.info("gaia data: " + str(gaia)) hsa_files = self.discard_files( resources_manager().get_list("/data/output/sources/hsa")) #log.info("hsa filess: " + str(hsa_files)) hsa = resources_manager().get_info(hsa_files, "/data/output/sources/hsa") #log.info("hsa data: " + str(hsa)) sim_files = self.discard_files( resources_manager().get_list("/data/output/sources/sim")) #log.info("hsa filess: " + str(hsa_files)) sim = resources_manager().get_info(sim_files, "/data/output/sources/sim") #log.info("hsa data: " + str(hsa)) results_files = self.discard_files( resources_manager().get_list("/data/output/results")) #log.info("results filess: " + str(results_files)) results_data = resources_manager().get_info(results_files, "/data/output/results") #log.info("results data: " + str(results_data)) user_files = self.discard_files( resources_manager().get_list("/data/output/user")) #log.info("user filess: " + str(user_files)) user_data = resources_manager().get_info(user_files, "/data/output/user") #log.info("user data: " + str(user_data)) #--------------------------------------------------------------------------------------------------- all_files.sort() pg = Paginator(all_files, wh.MAX_RESOURCES_PER_PAGE) page = wh.CURRENT_RESOURCES_PAGE if page < 1: wh.CURRENT_RESOURCES_PAGE = 1 elif page > pg.num_pages: wh.CURRENT_RESOURCES_PAGE = pg.num_pages files = pg.page(wh.CURRENT_RESOURCES_PAGE) f, d = resources_manager().get_info(files, wh.CURRENT_PATH) log.info(f) log.info(sorting_wh) if sorting_wh == 'size': f = collections.OrderedDict(sorted(f.items(), key=lambda x: x[1])) d = collections.OrderedDict(sorted(d.items(), key=lambda x: x[1])) elif sorting_wh == 'name': f = collections.OrderedDict(sorted(f.items(), key=lambda x: x[0])) d = collections.OrderedDict(sorted(d.items(), key=lambda x: x[0])) #Parse for the filemanager breadcrumb p = wh.CURRENT_PATH path_to_eliminate = gwh.RESOURCES_PATH #path_to_eliminate = re.sub("/results", '', path_to_eliminate) fail #p = gwh.RESULTS_PATH p = re.sub(path_to_eliminate, '', p) #p = path_to_eliminate #End parse for the filemanager breadcrumb p = p.split("/") self.job_data.data = [f, d, p, gaia, hsa, sim, results_data, user_data] self.job_data.ok = (pg.num_pages, wh.CURRENT_RESOURCES_PAGE, \ wh.CURRENT_RESOURCES_PAGE + 1, wh.CURRENT_RESOURCES_PAGE - 1) return self.job_data
def start(self, data): """This method runs the get_pipeline_status job. This method will retrieve the algorithm_models and it will sort them by the current sorting method provided by the wh_frontend_config warehouse. Then it will paginate the results with current page retrieved from the wh_frontend_config warehouse. Args: self: The object pointer. data: A dictorianry containing the input data for the job. Returns: The job_data attribute. The ok attribute provides the pages information. @see algorithm_model @link avi.models.algorithm_model @see wh_frontend_config @link avi.warehouse.wh_frontend_config """ log = logger().get_log("get_pipeline_status") wh = wh_frontend_config().get() sorting_wh = wh.SORTING_EXEC_BY order_by = 'request__pipeline_state__started_time' if sorting_wh == 'name': order_by = 'alg_name' elif sorting_wh == '-name': order_by = '-alg_name' elif sorting_wh == '-date': order_by = '-request__pipeline_state__started_time' elif sorting_wh == 'status': order_by = 'request__pipeline_state__state' elif sorting_wh == '-status': order_by = '-request__pipeline_state__state' all_ms = algorithm_model.objects.all().order_by(order_by, 'pk') self.job_data.data = {} self.job_data.ok = all_ms is not None if not all_ms: self.job_data.ok = False return self.job_data pg = Paginator(all_ms, wh.MAX_EXEC_PER_PAGE) page = wh.CURRENT_EXEC_PAGE #------------------ #----all data algorithms alldata = {} k = 0 for h in pg.object_list: try: status = h.request.pipe_state.state date = h.request.pipe_state.started_time except AttributeError: status = h.request.pipeline_state.state date = h.request.pipeline_state.started_time #status = h.request.pipeline_state.state #date = h.request.pipeline_state.started_time params = {} params['algorithm'] = {'name': h.alg_name, 'params': {}} ainfo_ms = algorithm_info_model.objects.filter(name=h.alg_name)[0] if ainfo_ms: qparams = literal_eval(h.params) mng = algorithm_manager() ainfo = mng.get_info(ainfo_ms.definition_file, 'input') for l in ainfo: if 'view_name' in ainfo[l]: params['algorithm']['params'][ainfo[l][ 'view_name']] = qparams['algorithm']['params'][ ainfo[l]['name']] else: params['algorithm']['params'][ ainfo[l]['name']] = qparams['algorithm']['params'][ ainfo[l]['name']] params = str(params) else: params = h.params alldata[k] = (h.alg_name, h.pk, params, date, status) log.info(params) k += 1 #------------------ if page < 1: wh.CURRENT_EXEC_PAGE = 1 elif page > pg.num_pages: wh.CURRENT_EXEC_PAGE = pg.num_pages ms = pg.page(wh.CURRENT_EXEC_PAGE) self.job_data.data = {} self.job_data.ok = ms is not None if not ms: self.job_data.ok = False return self.job_data data = {} i = 0 for j in ms: name = j.alg_name params = {} params['algorithm'] = {'name': j.alg_name, 'params': {}} try: status = j.request.pipe_state.state date = j.request.pipe_state.started_time except AttributeError: status = j.request.pipeline_state.state date = j.request.pipeline_state.started_time error = j.request.pipeline_state.exception pos = error.rfind("Exception: ") error = error[pos + 11:] ainfo_ms = algorithm_info_model.objects.filter( name=j.alg_name)[0] if ainfo_ms: qparams = literal_eval(j.params) mng = algorithm_manager() ainfo = mng.get_info(ainfo_ms.definition_file, 'input') for k in ainfo: if 'view_name' in ainfo[k]: params['algorithm']['params'][ ainfo[k]['view_name']] = str( qparams['algorithm']['params'][ainfo[k] ['name']]) else: params['algorithm']['params'][ ainfo[k]['name']] = str( qparams['algorithm']['params'][ainfo[k] ['name']]) params = str(params) else: params = j.params #log.info(error) if not error or error == "": error = "OK" if j.is_aborted: error = "Aborted" data[i] = (name, status, date, error, j.pk, params) i += 1 self.job_data.ok = (pg.num_pages, wh.CURRENT_EXEC_PAGE, \ wh.CURRENT_EXEC_PAGE + 1, wh.CURRENT_EXEC_PAGE - 1, alldata, wh.SORTING_EXEC_BY) self.job_data.data = data return self.job_data # OLD sorted_index = sorted(data, key=lambda x: data[x][1], reverse=True) i = 0 for ind in sorted_index: self.job_data.data[i] = data[ind] i += 1 return self.job_data
def get_file_list(self): """Deprecated""" from avi.utils.resources_manager import resources_manager return resources_manager() \ .get_file_list(wh_frontend_config().get().CURRENT_PATH)
def start(self, data): """This method runs the get_algorithms job. If the algorithms are not loaded it will load them. Then it will retrieve the all algorithm_info_models and return the data from them. Args: self: The object pointer. data: A dictionary containing the input data for the job. Returns: The job_data attribute. The ok attribute will be True if there are algorithms retrieved, False otherwise. @see algorithm_info_model @link avi.models.algorithm_info_model """ log = logger().get_log('algorithm_manager') wh_f = wh_frontend_config().get() if not wh().get().ALGORITHMS_LOADED: from avi.core.algorithm.algorithm_manager import algorithm_manager algorithm_manager().init() wh().get().ALGORITHMS_LOADED = True from avi.models import algorithm_info_model, algorithm_group_model all_ms = algorithm_info_model.objects.all().order_by( 'name_view', 'name', 'pk') #sall_ms = sorted(all_ms, key = lambda x:(x.name_view is None, x)) self.job_data.data = {} self.job_data.ok = all_ms is not None if not all_ms: return self.job_data pg = Paginator(all_ms, wh_f.MAX_ALG_PER_PAGE) page = wh_f.CURRENT_ALG_PAGE if page < 1: wh_f.CURRENT_ALG_PAGE = 1 elif page > pg.num_pages: wh_f.CURRENT_ALG_PAGE = pg.num_pages ms = pg.page(wh_f.CURRENT_ALG_PAGE) all_ms_g = algorithm_group_model.objects.all().order_by( 'name_view', 'name', 'pk') pg_g = Paginator(all_ms_g, wh_f.MAX_ALG_PER_PAGE) page = wh_f.CURRENT_ALG_PAGE if page < 1: wh_f.CURRENT_ALG_PAGE = 1 elif page > pg.num_pages: wh_f.CURRENT_ALG_PAGE = pg.num_pages ms_g = pg_g.page(wh_f.CURRENT_ALG_PAGE) data = [] for g in ms_g: data.append({"group": g, "algorithms": []}) data.sort(key=lambda x: x["group"].position, reverse=False) for j in ms: for g in data: if j.algorithm_group == g["group"].name: g["algorithms"].append( (j.pk, j.name, j.name_view, j.algorithm_type, j.algorithm_group, j.position)) for g in data: g["algorithms"].sort(key=lambda x: x[4], reverse=False) for g in data: log.info(g["group"].name) log.info(g["group"].position) for a in g["algorithms"]: log.info(a[4]) log.info(a[1]) # OLD # data = {} # i = 0 # for j in ms: # data[i] = (j.pk, # j.name, # j.name_view, # j.algorithm_type) # i += 1 res = {} res["algorithms"] = data res["max_pages"] = pg.num_pages res["current_page"] = wh_f.CURRENT_ALG_PAGE res["next_page"] = wh_f.CURRENT_ALG_PAGE + 1 res["prev_page"] = wh_f.CURRENT_ALG_PAGE - 1 self.job_data.data = res return self.job_data
def get_algorithm_list(self): """Deprecated Returns the algorithm list This method returns a dictionary with the information of all the algorithms. Args: self: The object pointer. Returns: A dictionary with the information of all the algorithms. """ num_alg = wh_frontend_config().get().MAX_ALG_PER_PAGE current_page = wh_frontend_config().get().CURRENT_ALG_PAGE path = wh_global_config().get().ALGORITHM_PATH self.log.info("Reading algorithms data from %s", path) data = {} jm = json_manager() for f in os.listdir(path): if not f.endswith(".json"): continue self.log.info("File %s", f) name, fext = os.path.splitext(f) name = name + ".py" self.log.info("Checking file %s", name) if not os.path.isfile(path + name): continue self.log.info("Algorithm file found, reading data file now") jm.load(path + f) self.log.info("Data loaded: %s", jm.json_root) if not self.__check_json(jm.json_root): continue self.log.info("JSON checked correctly") alg_name = jm.json_root['algorithm']['name'] data[alg_name] = jm.json_root['algorithm'] data2 = { "alg_data": { "alg_1": { "input": { "input_1": { "name": "table name", "type": "table" }, "input_2": { "name": "float name", "type": "float" } }, "name": "algorithm 1", }, "alg_2": { "input": { "input_1": { "name": "bool name", "type": "bool" }, "input_2": { "name": "float name", "type": "float" } }, "name": "asdf 2", } } } ret = {"alg_data": data} return ret return self.__sort_dict(data)