def has_param_type(self, json, param_type): """Checks if the given json file has the given parameter type. It loads the json file, checks if it is a valid json file and then checks if the given parameter type is in it. Args: self: The object pointer. json: The json file to be read. param_type: The parameter type to be checked. Returns: True if the parameter type is in the json, False otherwise See: json_manager: avi.utils.data.json_manager.json_manager See also: __check_json: __check_json() """ try: jm = json_manager() jm.load(json) if not self.__check_json(jm.json_root): self.log.error("Not valid json file") return None aux = jm.json_root['algorithm']['input'] for k in aux: input_type = aux[k]['type'] if input_type == param_type: return True return False except: return None # OLD try: jm = json_manager() jm.load(json) if not self.__check_json(jm.json_root): self.log.error("Not valid json file") return None aux = jm.json_root['algorithm']['input'] for g in aux: inputs = aux[g] for k in inputs: input_type = inputs[k]['type'] if input_type == param_type: return True return False except: return None
def get_info(self, json, key): """This method returns a dictionary with the algorithm information. This methods loads a given json file, checks if it is a valid json and then returns a dictionary containing the algorithm information. Args: self: The object pointer. json_file: The json file to be read. Returns: A dictionay containing the algorithm information if it is a valid json file, None otherwise. See: json_manager: avi.utils.data.json_manager.json_manager See also: __check_json: __check_json() """ try: jm = json_manager() jm.load(json) if not self.__check_json(jm.json_root): self.log.error("Not algorithm 2") return None return jm.json_root['algorithm'][key] except: return None
def get_algorithm_info(self, data): """Deprecated? """ self.log.debug(str(data)) if not "algorithm" in data.keys(): self.log.error("Not algorithm") return None path = wh_global_config().get().ALGORITHM_PATH jm = json_manager() alg_name = str(data['algorithm'][0]) self.log.debug("Algoritm name %s", alg_name) if not os.path.isfile(path + alg_name + ".json"): self.log.error("Not algorithm 1") return None jm.load(path + alg_name + ".json") if not self.__check_json(jm.json_root): self.log.error("Not algorithm 2") return None ret = {"algorithm": {}} alg = ret['algorithm'] alg['name'] = alg_name alg['params'] = {} params = alg['params'] for k, v in jm.json_root['algorithm']['input'].items(): param_name = v['name'] post_name = alg_name + "_" + k self.log.debug("param_name %s post_name %s data %s", param_name, post_name, "") params[param_name] = self.__get_data(data[post_name][0], v['type']) data2 = { "algorithm": { "name": "dummy_algorithm", "params": { "param1": 1.0, #"float", "param2": 2.0, #"float", }, } } return ret
def get_algorithm_data(self, alg_id, name, def_file, post): """Returns a dictionary with all the algorithm parameters from a POST message The methods reads the given definition file and the given POST and creates a dictionary with the values of the post message. Args: self: The object pointer. alg_id: The algorithm's id. name: The algorithm's name. def_file: The algorithm definition file. post: The POST message containing all the values. Returns: A dictionary with all the algorithms parameters from the POST message, None if the definition file is not a valid json file. """ try: jm = json_manager() if not os.path.isfile(def_file): self.log.error("Not algorithm 1") return None jm.load(def_file) if not self.__check_json(jm.json_root): self.log.error("Not algorithm 2") return None ret = {"algorithm": {}} alg = ret['algorithm'] alg['name'] = name alg['params'] = {} params = alg['params'] for k, v in jm.json_root['algorithm']['input'].items(): param_name = v['name'] post_name = alg_id + "_" + param_name self.log.info("param_name %s post_name %s data %s", param_name, post_name, "") params[param_name] = self.__get_data(post[post_name][0], v['type']) return ret except: return None
def get_positional_sources_from_herschel(input_file): from avi.utils.data.json_manager import json_manager from avi.utils.data.file_manager import file_manager from avi.utils.coordinates_manager import coordinates_manager from avi.core.interface.name_solvers import simbad, ned jm = json_manager() fm = file_manager() cm = coordinates_manager() r = risea().get() im = r.interface_manager data = jm.read_herschel_input(input_file) c = 0 for i in data: c += 1 try: ra = None dec = None if i.get('name'): coords = simbad().get_object_coordinates(i['name']) if not coords: coords = ned().get_object_coordinates(i['name']) if not coords: continue v_ra = coords['ra'] v_dec = coords['dec'] else: v_ra = i.get('ra') v_dec = i.get('dec') if not v_ra or not v_dec: v_l = i.get('l') v_b = i.get('b') if not v_l or not v_b: continue coords = cm.gal_to_icrs(float(v_l), float(v_b)) ra = coords['ra'] dec = coords['dec'] else: try: ra = float(v_ra) dec = float(v_dec) except ValueError: coords = cm.icrs_degrees(v_ra, v_dec) ra = coords['ra'] dec = coords['dec'] table = None src = None wl = int(i['wavelength']) if wl == 70 or wl == 100 or wl == 160: table = "cat_hppsc_%s" % (str(wl).zfill(3)) elif wl == 250 or wl == 350 or wl == 500: table = "cat_spsc_%i" % (wl) if i['shape'] == 'cone': radius = float(i['radius']) src = im._archive_herschel_get_circle(ra, dec, radius, table) elif i['shape'] == 'box': width = float(i['width']) height = float(i['height']) src = im._archive_herschel_get_box(ra, dec, width, height, table) elif i['shape'] == 'polygon': vertexes = jm.get_vertexes(i) src = im._archive_herschel_get_polygon(ra, dec, vertexes, table) if src != None: if i.get('output_file'): fm.save_file_plain_data(src, "%s.vot" % (i['output_file'])) else: fm.save_file_plain_data(src, "source_%i.vot" % (c)) except ValueError as e: print("Value Error") print(traceback.format_exc()) pass except Exception as e: print("Exception") print(traceback.format_exc()) pass
def get_maps_from_herschel(input_file): from avi.utils.data.json_manager import json_manager from avi.utils.data.file_manager import file_manager from avi.utils.coordinates_manager import coordinates_manager from avi.core.interface.name_solvers import simbad, ned jm = json_manager() fm = file_manager() cm = coordinates_manager() r = risea().get() im = r.interface_manager data = jm.read_herschel_input(input_file) c = 0 for i in data: c += 1 try: ra = None dec = None if i.get('name'): coords = simbad().get_object_coordinates(i['name']) if not coords: coords = ned().get_object_coordinates(i['name']) if not coords: continue v_ra = coords['ra'] v_dec = coords['dec'] else: v_ra = i.get('ra') v_dec = i.get('dec') if not v_ra or not v_dec: v_l = i.get('l') v_b = i.get('b') if not v_l or not v_b: continue coords = cm.gal_to_icrs(float(v_l), float(v_b)) ra = coords['ra'] dec = coords['dec'] else: try: ra = float(v_ra) dec = float(v_dec) except ValueError: coords = cm.icrs_degrees(v_ra, v_dec) ra = coords['ra'] dec = coords['dec'] table = None src = None wl = int(i['wavelength']) if wl == 70 or wl == 100 or wl == 160: table = "cat_hppsc_%s" % (str(wl).zfill(3)) elif wl == 250 or wl == 350 or wl == 500: table = "cat_spsc_%i" % (wl) radius = float(i['size']) im.archive_get_maps(ra, dec, radius, 'All', 'PACS', table=table) except ValueError as e: print("Value Error") print(traceback.format_exc()) pass except Exception as e: print("Exception") print(traceback.format_exc()) pass
def get_herschel_data(self, log, data): """Does a query to the herschel archive It will read the input contained in the data parameter and it will query the herschel archive trough the interface_manager. Then it will save the results using the file_manager Args: self: The object pointer log: The log data: The input data to the query Raises: task_exception: avi.task.task.task_exception See: interface_manager: avi.core.interface.interface_manager.interface_manager See also: file_manager: avi.utils.data.file_manager.file_manager """ log.debug('get_herschel_data method') im = risea().get().interface_manager fm = file_manager() cm = coordinates_manager() jm = json_manager() if not im: log.error('There is no interface manager initialized!') raise err("There is no interface manager initialized!") try: ra = None dec = None if data.get('name') and data.get('name_coord') == 'name': log.info("Name attr %s found, retrieving coordinates from " \ + "Simbad/Ned databases", data['name']) coords = simbad().get_object_coordinates(data['name']) if not coords: coords = ned().get_object_coordinates(data['name']) if not coords: log.error('Name %s not found in Simbad/Ned data bases', data['name']) raise err('Name %s not found in Simbad/Ned data bases', data['name']) v_ra = coords['ra'] v_dec = coords['dec'] else: log.info("Retrieving coordinates from the provided data...") v_ra = data.get('ra') v_dec = data.get('dec') if not v_ra or not v_dec: log.info("No equatorial coordinates found!") log.info("Reading galactic coordinates from data...") v_l = data.get('l') v_b = data.get('b') if not v_l or not v_b: log.error('No valid coordinates found') raise err('No valid coordinates found') coords = cm.gal_to_icrs(float(v_l), float(v_b)) ra = coords['ra'] dec = coords['dec'] else: try: ra = float(v_ra) dec = float(v_dec) except ValueError: coords = cm.icrs_degrees(v_ra, v_dec) ra = coords['ra'] dec = coords['dec'] src = None shape = data['shape'] if shape != 'cone' and shape != 'box' and shape != 'polygon': log.error("Unknown shape!") raise err("Unknown shape!") log.info("Shape: %s", shape) table = data['table'] if not table or table == "": table = "v_active_observation" log.info("Table: %s", "hsa.%s" % (table)) if not data['positional_images']: log.info("Retrieving positional sources from " + "the herschel archive...") if shape == 'cone': if not data['radius']: log.error("No radius provided") raise err("No radius provided") src = im._archive_herschel_get_circle( ra, dec, data['radius'], table) elif shape == 'box': if not data['width'] or not data['height']: log.error("No dimensions provided") raise err("No dimansions provided") src = im._archive_herschel_get_box(ra, dec, data['width'], data['height'], table) elif shape == 'polygon': vertexes = jm.get_vertexes(data) src = im._archive_herschel_get_polygon( ra, dec, vertexes, table) if src != None: if not data.get('output_file'): file_name = wh().get() \ .SOURCES_FMT%{"mission":"hsa", "date":str(round(time.time())), "name":"data"} else: file_name = wh().get() \ .SOURCES_FMT%{"mission":"hsa", "date":str(round(time.time())), "name":data['output_file']} fm.save_file_plain_data(src, "%s.vot" % (file_name), wh().get().HSA_PATH, self.task_id, "hsa", timezone.now()) #"%f_%f_%s_%s.vot" \ #%(ra,dec,shape,table)) else: log.error( "Something went wrong while querying the archive!") raise err( "Something went wrong while querying the archive!") log.info("Everything done!") return log.info('Retrieving maps from the herschel archive...') if shape == 'cone': if not data['radius']: log.error("No radius provided") raise err("No radius provided") if not data.get('output_file'): im.archive_get_maps(ra, dec, data['radius'], data['level'], data['instrument'], id=self.task_id) else: log.info("fileeeeeeeeeee") im.archive_get_maps(ra, dec, data['radius'], data['level'], data['instrument'], id=self.task_id, name=data['output_file']) log.info("Everything done!") return elif shape == 'box': if not data['width'] or not data['height']: log.error("No dimensions provided") raise err("No dimansions provided") if not data.get('output_file'): im.archive_get_maps_box(ra, dec, data['width'], data['height'], data['level'], data['instrument'], id=self.task_id) else: im.archive_get_maps_box(ra, dec, data['width'], data['height'], data['level'], data['instrument'], id=self.task_id, name=data['output_file']) log.info("Everything done!") return elif shape == 'polygon': vertexes = jm.get_vertexes(data) if not data.get('output_file'): im.archive_get_maps_polygon(ra, dec, vertexes, data['level'], data['instrument'], id=self.task_id) else: im.archive_get_maps_polygon(ra, dec, vertexes, data['level'], data['instrument'], id=self.task_id, name=data['output_file']) log.info("Everything done!") return log.error("Something went wrong...") raise err("Something went wrong...") except Exception: log.error(traceback.format_exc()) raise err(traceback.format_exc())
def run(self): """Runs the query to the herschel archive. If the task_data contains the 'input_file' key it will read that value and call get_herschel_data() once per input parameter found in the input_file. If the task_data contains the 'adql' key it will query the archive through the interface_manager using that query. Otherwise it will call get_herschel_data() with the input from task_data Args: self: The object pointer. Raises: task_exception: avi.task.task.task_exception See: interface_manager: avi.core.interface.interface_manager.interface_manager See also: get_herschel_data: get_herschel_data() """ def get_herschel_data(log, data): """Deprecated""" pass log = logger().get_log('herschel_query_task') data = self.task_data.data jm = json_manager() if data.get('input_file') and data.get('name_coord') == 'file': log.info('There is an input file') #self.get_herschel_data(log, data) try: d = jm.read_herschel_input(data['input_file']) for i in d: if i.get('name'): i['name_coord'] = 'name' if i.get('wavelength'): wl = int(i['wavelength']) if wl == 70 or wl == 100 or wl == 160: i['tablee'] = "hsa.pacs_point_source_%s" % ( str(wl).zfill(3)) #"cat_hppsc_%s"%(str(wl).zfill(3)) elif wl == 250 or wl == 350 or wl == 500: i['tablee'] = "hsa.spire_point_source_%s" % (wl) #"cat_spsc_%i"%(wl) if i.get('positional_source'): if i['positional_source'] == 'False': i['positional_images'] = True else: i['positional_images'] = False else: i['positional_images'] = True self.get_herschel_data(log, i) except Exception: log.error("Exception while retrieving data from herschel") log.error(traceback.format_exc()) raise err(traceback.format_exc()) finally: pass #os.remove(data['input_file']) return elif data.get('adql') and data.get('name_coord') == 'adql': log.info('ADQL query') im = risea().get().interface_manager fm = file_manager() adql = data['adql'] if not im: log.error('There is no interface manager initialized!') raise err("There is no interface manager initialized!") src = im._archive_herschel_get_adql(adql) if src != None: if not data.get('output_file'): file_name = wh().get().SOURCES_FMT % { "mission": "hsa", "date": str(round(time.time())), "name": "data" } else: file_name = wh().get().SOURCES_FMT % { "mission": "hsa", "date": str(round(time.time())), "name": data['output_file'] } fm.save_file_plain_data(src, "%s.vot" % (file_name), wh().get().HSA_PATH, self.task_id, "hsa", timezone.now()) log.info("Everything done!") return else: if data.get('shape') == 'polygon': jm.set_vertexes(data, data['polygon']) log.info("added vertexes %s", str(data)) self.get_herschel_data(log, data) return
def run(self): """Runs the query to the gaia archive. If the task_data contains the 'input_file' key it will read that value and call get_gaia_data() once per input parameter found in the input_file. If the task_data contains the 'adql' key it will query the archive through the interface_manager using that query. Otherwise it will call get_gaia_data() with the input from task_data Args: self: The object pointer. Raises: task_exception: avi.task.task.task_exception See: interface_manager: avi.core.interface.interface_manager.interface_manager See also: get_gaia_data: get_gaia_data() """ log = logger().get_log('gaia_query_task') jm = json_manager() data = self.task_data.data log.info("%s", str(data)) if data.get('input_file') and data.get('name_coord') == 'file': log.info('There is an input file') try: d = jm.read_gaia_input(data['input_file']) for i in d: if i.get('name'): i['name_coord'] = 'name' self.get_gaia_data(log, i) except Exception: log.error("Exception while retrieving data from gaia") log.error(traceback.format_exc()) raise err(traceback.format_exc()) finally: os.remove(data['input_file']) return elif data.get('adql') and data.get('name_coord') == 'adql': log.info('ADQL query') im = risea().get().interface_manager fm = file_manager() adql = data['adql'] if not im: log.error('There is no interface manager initialized!') raise err("There is no interface manager initialized!") src = im._archive_gaia_get_adql(adql) if src != None: if not data.get('output_file'): file_name = wh().get().SOURCES_FMT % { "mission": "gaia", "date": str(round(time.time())), "name": "data" } else: file_name = wh().get().SOURCES_FMT % { "mission": "gaia", "date": str(round(time.time())), "name": data['output_file'] } fm.save_file_plain_data(src, "%s.vot" % (file_name), wh().get().GAIA_PATH, self.task_id, "gaia", timezone.now()) log.info("Everything done!") return else: if data.get('shape') == 'polygon': jm.set_vertexes(data, data['polygon']) self.get_gaia_data(log, data) return
def update_database(self, path, alg_type): """This method updates the algorithms in the database. This method will read files in the given directory and if they are any it will store the information in algorithm_info_models. Args: self: The object pointer. path: The directory path to be read. alg_type: The type of the algorithms in the given directory. See: algorithm_info_model @link avi.models.algorithm_info_model """ # read folder alg = {} for f in os.listdir(path): if not f.endswith(".json"): continue name, fext = os.path.splitext(f) if name == "groups": try: jm = json_manager() jm.load(os.path.join(path, f)) for _, g in jm.json_root["groups"].items(): m = algorithm_group_model.objects.all().filter( name=g["name"]) if not m: if not "view_name" in g: g["view_name"] = g["name"] m = algorithm_group_model(name=g["name"], position=g["position"], name_view=g["view_name"]) m.save() except Exception as err: self.log.info(err) continue name_src = name + ".py" if not os.path.isfile(os.path.join(path, name_src)): continue alg[name] = { "name": name, "name_view": self.get_info(os.path.join(path, f), "view_name"), "src": os.path.join(path, name_src), "json": os.path.join(path, f), "type": alg_type, "group": self.get_info(os.path.join(path, f), "group"), "position": self.get_info(os.path.join(path, f), "position") } # check and add new ones #self.log.info(alg) for name in alg: #self.log.info(str(alg[name])) m = algorithm_info_model.objects.all().filter(name=name) #self.log.info("after query %s", str(m)) #self.log.info("after query") if not m: self.log.info("not m") a = alg[name] if not a["name_view"]: a["name_view"] = a["name"] m = algorithm_info_model(name=a["name"], source_file=a["src"], name_view=a["name_view"], definition_file=a["json"], algorithm_type=a["type"], algorithm_group=a["group"], position=a["position"]) m.save()
def get_algorithm_list(self): """Deprecated Returns the algorithm list This method returns a dictionary with the information of all the algorithms. Args: self: The object pointer. Returns: A dictionary with the information of all the algorithms. """ num_alg = wh_frontend_config().get().MAX_ALG_PER_PAGE current_page = wh_frontend_config().get().CURRENT_ALG_PAGE path = wh_global_config().get().ALGORITHM_PATH self.log.info("Reading algorithms data from %s", path) data = {} jm = json_manager() for f in os.listdir(path): if not f.endswith(".json"): continue self.log.info("File %s", f) name, fext = os.path.splitext(f) name = name + ".py" self.log.info("Checking file %s", name) if not os.path.isfile(path + name): continue self.log.info("Algorithm file found, reading data file now") jm.load(path + f) self.log.info("Data loaded: %s", jm.json_root) if not self.__check_json(jm.json_root): continue self.log.info("JSON checked correctly") alg_name = jm.json_root['algorithm']['name'] data[alg_name] = jm.json_root['algorithm'] data2 = { "alg_data": { "alg_1": { "input": { "input_1": { "name": "table name", "type": "table" }, "input_2": { "name": "float name", "type": "float" } }, "name": "algorithm 1", }, "alg_2": { "input": { "input_1": { "name": "bool name", "type": "bool" }, "input_2": { "name": "float name", "type": "float" } }, "name": "asdf 2", } } } ret = {"alg_data": data} return ret return self.__sort_dict(data)