def has_param_type(self, json, param_type):
"""Checks if the given json file has the given parameter type.
It loads the json file, checks if it is a valid json file and then
checks if the given parameter type is in it.
Args:
self: The object pointer.
json: The json file to be read.
param_type: The parameter type to be checked.
Returns:
True if the parameter type is in the json, False otherwise
See:
json_manager: avi.utils.data.json_manager.json_manager
See also:
__check_json: __check_json()
"""
try:
jm = json_manager()
jm.load(json)
if not self.__check_json(jm.json_root):
self.log.error("Not valid json file")
return None
aux = jm.json_root['algorithm']['input']
for k in aux:
input_type = aux[k]['type']
if input_type == param_type:
return True
return False
except:
return None
# OLD
try:
jm = json_manager()
jm.load(json)
if not self.__check_json(jm.json_root):
self.log.error("Not valid json file")
return None
aux = jm.json_root['algorithm']['input']
for g in aux:
inputs = aux[g]
for k in inputs:
input_type = inputs[k]['type']
if input_type == param_type:
return True
return False
except:
return None
def get_info(self, json, key):
"""This method returns a dictionary with the algorithm information.
This methods loads a given json file, checks if it is a valid json and
then returns a dictionary containing the algorithm information.
Args:
self: The object pointer.
json_file: The json file to be read.
Returns:
A dictionay containing the algorithm information if it is a valid json
file, None otherwise.
See:
json_manager: avi.utils.data.json_manager.json_manager
See also:
__check_json: __check_json()
"""
try:
jm = json_manager()
jm.load(json)
if not self.__check_json(jm.json_root):
self.log.error("Not algorithm 2")
return None
return jm.json_root['algorithm'][key]
except:
return None
def get_algorithm_info(self, data):
"""Deprecated?
"""
self.log.debug(str(data))
if not "algorithm" in data.keys():
self.log.error("Not algorithm")
return None
path = wh_global_config().get().ALGORITHM_PATH
jm = json_manager()
alg_name = str(data['algorithm'][0])
self.log.debug("Algoritm name %s", alg_name)
if not os.path.isfile(path + alg_name + ".json"):
self.log.error("Not algorithm 1")
return None
jm.load(path + alg_name + ".json")
if not self.__check_json(jm.json_root):
self.log.error("Not algorithm 2")
return None
ret = {"algorithm": {}}
alg = ret['algorithm']
alg['name'] = alg_name
alg['params'] = {}
params = alg['params']
for k, v in jm.json_root['algorithm']['input'].items():
param_name = v['name']
post_name = alg_name + "_" + k
self.log.debug("param_name %s post_name %s data %s", param_name,
post_name, "")
params[param_name] = self.__get_data(data[post_name][0], v['type'])
data2 = {
"algorithm": {
"name": "dummy_algorithm",
"params": {
"param1": 1.0, #"float",
"param2": 2.0, #"float",
},
}
}
return ret
def get_algorithm_data(self, alg_id, name, def_file, post):
"""Returns a dictionary with all the algorithm parameters from a POST
message
The methods reads the given definition file and the given POST and
creates a dictionary with the values of the post message.
Args:
self: The object pointer.
alg_id: The algorithm's id.
name: The algorithm's name.
def_file: The algorithm definition file.
post: The POST message containing all the values.
Returns:
A dictionary with all the algorithms parameters from the POST message,
None if the definition file is not a valid json file.
"""
try:
jm = json_manager()
if not os.path.isfile(def_file):
self.log.error("Not algorithm 1")
return None
jm.load(def_file)
if not self.__check_json(jm.json_root):
self.log.error("Not algorithm 2")
return None
ret = {"algorithm": {}}
alg = ret['algorithm']
alg['name'] = name
alg['params'] = {}
params = alg['params']
for k, v in jm.json_root['algorithm']['input'].items():
param_name = v['name']
post_name = alg_id + "_" + param_name
self.log.info("param_name %s post_name %s data %s", param_name,
post_name, "")
params[param_name] = self.__get_data(post[post_name][0],
v['type'])
return ret
except:
return None
def get_positional_sources_from_herschel(input_file):
from avi.utils.data.json_manager import json_manager
from avi.utils.data.file_manager import file_manager
from avi.utils.coordinates_manager import coordinates_manager
from avi.core.interface.name_solvers import simbad, ned
jm = json_manager()
fm = file_manager()
cm = coordinates_manager()
r = risea().get()
im = r.interface_manager
data = jm.read_herschel_input(input_file)
c = 0
for i in data:
c += 1
try:
ra = None
dec = None
if i.get('name'):
coords = simbad().get_object_coordinates(i['name'])
if not coords:
coords = ned().get_object_coordinates(i['name'])
if not coords:
continue
v_ra = coords['ra']
v_dec = coords['dec']
else:
v_ra = i.get('ra')
v_dec = i.get('dec')
if not v_ra or not v_dec:
v_l = i.get('l')
v_b = i.get('b')
if not v_l or not v_b:
continue
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
table = None
src = None
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
table = "cat_hppsc_%s" % (str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
table = "cat_spsc_%i" % (wl)
if i['shape'] == 'cone':
radius = float(i['radius'])
src = im._archive_herschel_get_circle(ra, dec, radius, table)
elif i['shape'] == 'box':
width = float(i['width'])
height = float(i['height'])
src = im._archive_herschel_get_box(ra, dec, width, height,
table)
elif i['shape'] == 'polygon':
vertexes = jm.get_vertexes(i)
src = im._archive_herschel_get_polygon(ra, dec, vertexes,
table)
if src != None:
if i.get('output_file'):
fm.save_file_plain_data(src, "%s.vot" % (i['output_file']))
else:
fm.save_file_plain_data(src, "source_%i.vot" % (c))
except ValueError as e:
print("Value Error")
print(traceback.format_exc())
pass
except Exception as e:
print("Exception")
print(traceback.format_exc())
pass
def get_maps_from_herschel(input_file):
from avi.utils.data.json_manager import json_manager
from avi.utils.data.file_manager import file_manager
from avi.utils.coordinates_manager import coordinates_manager
from avi.core.interface.name_solvers import simbad, ned
jm = json_manager()
fm = file_manager()
cm = coordinates_manager()
r = risea().get()
im = r.interface_manager
data = jm.read_herschel_input(input_file)
c = 0
for i in data:
c += 1
try:
ra = None
dec = None
if i.get('name'):
coords = simbad().get_object_coordinates(i['name'])
if not coords:
coords = ned().get_object_coordinates(i['name'])
if not coords:
continue
v_ra = coords['ra']
v_dec = coords['dec']
else:
v_ra = i.get('ra')
v_dec = i.get('dec')
if not v_ra or not v_dec:
v_l = i.get('l')
v_b = i.get('b')
if not v_l or not v_b:
continue
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
table = None
src = None
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
table = "cat_hppsc_%s" % (str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
table = "cat_spsc_%i" % (wl)
radius = float(i['size'])
im.archive_get_maps(ra, dec, radius, 'All', 'PACS', table=table)
except ValueError as e:
print("Value Error")
print(traceback.format_exc())
pass
except Exception as e:
print("Exception")
print(traceback.format_exc())
pass
def get_herschel_data(self, log, data):
"""Does a query to the herschel archive
It will read the input contained in the data parameter and it will
query the herschel archive trough the interface_manager. Then it will
save the results using the file_manager
Args:
self: The object pointer
log: The log
data: The input data to the query
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
file_manager: avi.utils.data.file_manager.file_manager
"""
log.debug('get_herschel_data method')
im = risea().get().interface_manager
fm = file_manager()
cm = coordinates_manager()
jm = json_manager()
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
try:
ra = None
dec = None
if data.get('name') and data.get('name_coord') == 'name':
log.info("Name attr %s found, retrieving coordinates from " \
+ "Simbad/Ned databases", data['name'])
coords = simbad().get_object_coordinates(data['name'])
if not coords:
coords = ned().get_object_coordinates(data['name'])
if not coords:
log.error('Name %s not found in Simbad/Ned data bases',
data['name'])
raise err('Name %s not found in Simbad/Ned data bases',
data['name'])
v_ra = coords['ra']
v_dec = coords['dec']
else:
log.info("Retrieving coordinates from the provided data...")
v_ra = data.get('ra')
v_dec = data.get('dec')
if not v_ra or not v_dec:
log.info("No equatorial coordinates found!")
log.info("Reading galactic coordinates from data...")
v_l = data.get('l')
v_b = data.get('b')
if not v_l or not v_b:
log.error('No valid coordinates found')
raise err('No valid coordinates found')
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
src = None
shape = data['shape']
if shape != 'cone' and shape != 'box' and shape != 'polygon':
log.error("Unknown shape!")
raise err("Unknown shape!")
log.info("Shape: %s", shape)
table = data['table']
if not table or table == "":
table = "v_active_observation"
log.info("Table: %s", "hsa.%s" % (table))
if not data['positional_images']:
log.info("Retrieving positional sources from " +
"the herschel archive...")
if shape == 'cone':
if not data['radius']:
log.error("No radius provided")
raise err("No radius provided")
src = im._archive_herschel_get_circle(
ra, dec, data['radius'], table)
elif shape == 'box':
if not data['width'] or not data['height']:
log.error("No dimensions provided")
raise err("No dimansions provided")
src = im._archive_herschel_get_box(ra, dec, data['width'],
data['height'], table)
elif shape == 'polygon':
vertexes = jm.get_vertexes(data)
src = im._archive_herschel_get_polygon(
ra, dec, vertexes, table)
if src != None:
if not data.get('output_file'):
file_name = wh().get() \
.SOURCES_FMT%{"mission":"hsa",
"date":str(round(time.time())),
"name":"data"}
else:
file_name = wh().get() \
.SOURCES_FMT%{"mission":"hsa",
"date":str(round(time.time())),
"name":data['output_file']}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().HSA_PATH, self.task_id,
"hsa", timezone.now())
#"%f_%f_%s_%s.vot" \
#%(ra,dec,shape,table))
else:
log.error(
"Something went wrong while querying the archive!")
raise err(
"Something went wrong while querying the archive!")
log.info("Everything done!")
return
log.info('Retrieving maps from the herschel archive...')
if shape == 'cone':
if not data['radius']:
log.error("No radius provided")
raise err("No radius provided")
if not data.get('output_file'):
im.archive_get_maps(ra,
dec,
data['radius'],
data['level'],
data['instrument'],
id=self.task_id)
else:
log.info("fileeeeeeeeeee")
im.archive_get_maps(ra,
dec,
data['radius'],
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
elif shape == 'box':
if not data['width'] or not data['height']:
log.error("No dimensions provided")
raise err("No dimansions provided")
if not data.get('output_file'):
im.archive_get_maps_box(ra,
dec,
data['width'],
data['height'],
data['level'],
data['instrument'],
id=self.task_id)
else:
im.archive_get_maps_box(ra,
dec,
data['width'],
data['height'],
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
elif shape == 'polygon':
vertexes = jm.get_vertexes(data)
if not data.get('output_file'):
im.archive_get_maps_polygon(ra,
dec,
vertexes,
data['level'],
data['instrument'],
id=self.task_id)
else:
im.archive_get_maps_polygon(ra,
dec,
vertexes,
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
log.error("Something went wrong...")
raise err("Something went wrong...")
except Exception:
log.error(traceback.format_exc())
raise err(traceback.format_exc())
def run(self):
"""Runs the query to the herschel archive.
If the task_data contains the 'input_file' key it will read that value
and call get_herschel_data() once per input parameter found in the
input_file.
If the task_data contains the 'adql' key it will query the archive
through the interface_manager using that query.
Otherwise it will call get_herschel_data() with the input from task_data
Args:
self: The object pointer.
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
get_herschel_data: get_herschel_data()
"""
def get_herschel_data(log, data):
"""Deprecated"""
pass
log = logger().get_log('herschel_query_task')
data = self.task_data.data
jm = json_manager()
if data.get('input_file') and data.get('name_coord') == 'file':
log.info('There is an input file')
#self.get_herschel_data(log, data)
try:
d = jm.read_herschel_input(data['input_file'])
for i in d:
if i.get('name'):
i['name_coord'] = 'name'
if i.get('wavelength'):
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
i['tablee'] = "hsa.pacs_point_source_%s" % (
str(wl).zfill(3))
#"cat_hppsc_%s"%(str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
i['tablee'] = "hsa.spire_point_source_%s" % (wl)
#"cat_spsc_%i"%(wl)
if i.get('positional_source'):
if i['positional_source'] == 'False':
i['positional_images'] = True
else:
i['positional_images'] = False
else:
i['positional_images'] = True
self.get_herschel_data(log, i)
except Exception:
log.error("Exception while retrieving data from herschel")
log.error(traceback.format_exc())
raise err(traceback.format_exc())
finally:
pass
#os.remove(data['input_file'])
return
elif data.get('adql') and data.get('name_coord') == 'adql':
log.info('ADQL query')
im = risea().get().interface_manager
fm = file_manager()
adql = data['adql']
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
src = im._archive_herschel_get_adql(adql)
if src != None:
if not data.get('output_file'):
file_name = wh().get().SOURCES_FMT % {
"mission": "hsa",
"date": str(round(time.time())),
"name": "data"
}
else:
file_name = wh().get().SOURCES_FMT % {
"mission": "hsa",
"date": str(round(time.time())),
"name": data['output_file']
}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().HSA_PATH, self.task_id,
"hsa", timezone.now())
log.info("Everything done!")
return
else:
if data.get('shape') == 'polygon':
jm.set_vertexes(data, data['polygon'])
log.info("added vertexes %s", str(data))
self.get_herschel_data(log, data)
return
def run(self):
"""Runs the query to the gaia archive.
If the task_data contains the 'input_file' key it will read that value
and call get_gaia_data() once per input parameter found in the
input_file.
If the task_data contains the 'adql' key it will query the archive
through the interface_manager using that query.
Otherwise it will call get_gaia_data() with the input from task_data
Args:
self: The object pointer.
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
get_gaia_data: get_gaia_data()
"""
log = logger().get_log('gaia_query_task')
jm = json_manager()
data = self.task_data.data
log.info("%s", str(data))
if data.get('input_file') and data.get('name_coord') == 'file':
log.info('There is an input file')
try:
d = jm.read_gaia_input(data['input_file'])
for i in d:
if i.get('name'):
i['name_coord'] = 'name'
self.get_gaia_data(log, i)
except Exception:
log.error("Exception while retrieving data from gaia")
log.error(traceback.format_exc())
raise err(traceback.format_exc())
finally:
os.remove(data['input_file'])
return
elif data.get('adql') and data.get('name_coord') == 'adql':
log.info('ADQL query')
im = risea().get().interface_manager
fm = file_manager()
adql = data['adql']
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
src = im._archive_gaia_get_adql(adql)
if src != None:
if not data.get('output_file'):
file_name = wh().get().SOURCES_FMT % {
"mission": "gaia",
"date": str(round(time.time())),
"name": "data"
}
else:
file_name = wh().get().SOURCES_FMT % {
"mission": "gaia",
"date": str(round(time.time())),
"name": data['output_file']
}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().GAIA_PATH, self.task_id,
"gaia", timezone.now())
log.info("Everything done!")
return
else:
if data.get('shape') == 'polygon':
jm.set_vertexes(data, data['polygon'])
self.get_gaia_data(log, data)
return
def update_database(self, path, alg_type):
"""This method updates the algorithms in the database.
This method will read files in the given directory and if they are any
it will store the information in algorithm_info_models.
Args:
self: The object pointer.
path: The directory path to be read.
alg_type: The type of the algorithms in the given directory.
See:
algorithm_info_model @link avi.models.algorithm_info_model
"""
# read folder
alg = {}
for f in os.listdir(path):
if not f.endswith(".json"):
continue
name, fext = os.path.splitext(f)
if name == "groups":
try:
jm = json_manager()
jm.load(os.path.join(path, f))
for _, g in jm.json_root["groups"].items():
m = algorithm_group_model.objects.all().filter(
name=g["name"])
if not m:
if not "view_name" in g:
g["view_name"] = g["name"]
m = algorithm_group_model(name=g["name"],
position=g["position"],
name_view=g["view_name"])
m.save()
except Exception as err:
self.log.info(err)
continue
name_src = name + ".py"
if not os.path.isfile(os.path.join(path, name_src)):
continue
alg[name] = {
"name": name,
"name_view": self.get_info(os.path.join(path, f), "view_name"),
"src": os.path.join(path, name_src),
"json": os.path.join(path, f),
"type": alg_type,
"group": self.get_info(os.path.join(path, f), "group"),
"position": self.get_info(os.path.join(path, f), "position")
}
# check and add new ones
#self.log.info(alg)
for name in alg:
#self.log.info(str(alg[name]))
m = algorithm_info_model.objects.all().filter(name=name)
#self.log.info("after query %s", str(m))
#self.log.info("after query")
if not m:
self.log.info("not m")
a = alg[name]
if not a["name_view"]:
a["name_view"] = a["name"]
m = algorithm_info_model(name=a["name"],
source_file=a["src"],
name_view=a["name_view"],
definition_file=a["json"],
algorithm_type=a["type"],
algorithm_group=a["group"],
position=a["position"])
m.save()
def get_algorithm_list(self):
"""Deprecated
Returns the algorithm list
This method returns a dictionary with the information of all the
algorithms.
Args:
self: The object pointer.
Returns:
A dictionary with the information of all the algorithms.
"""
num_alg = wh_frontend_config().get().MAX_ALG_PER_PAGE
current_page = wh_frontend_config().get().CURRENT_ALG_PAGE
path = wh_global_config().get().ALGORITHM_PATH
self.log.info("Reading algorithms data from %s", path)
data = {}
jm = json_manager()
for f in os.listdir(path):
if not f.endswith(".json"):
continue
self.log.info("File %s", f)
name, fext = os.path.splitext(f)
name = name + ".py"
self.log.info("Checking file %s", name)
if not os.path.isfile(path + name):
continue
self.log.info("Algorithm file found, reading data file now")
jm.load(path + f)
self.log.info("Data loaded: %s", jm.json_root)
if not self.__check_json(jm.json_root):
continue
self.log.info("JSON checked correctly")
alg_name = jm.json_root['algorithm']['name']
data[alg_name] = jm.json_root['algorithm']
data2 = {
"alg_data": {
"alg_1": {
"input": {
"input_1": {
"name": "table name",
"type": "table"
},
"input_2": {
"name": "float name",
"type": "float"
}
},
"name": "algorithm 1",
},
"alg_2": {
"input": {
"input_1": {
"name": "bool name",
"type": "bool"
},
"input_2": {
"name": "float name",
"type": "float"
}
},
"name": "asdf 2",
}
}
}
ret = {"alg_data": data}
return ret
return self.__sort_dict(data)