def get_info(self, data, path):
"""Returns the files and directories information
Returns the files and directories inforamtion within the given files
contained in 'data' and the given path
Args:
self: The object pointer
data: The files which information has to be retrieved
path: The path
Returns:
All the files and directories informatin within 'data' and 'path'
"""
directories = collections.OrderedDict()
files = collections.OrderedDict()
fm = file_manager()
for f in data:
if os.path.isdir(os.path.join(path, f)):
directories[f] = self. \
convert_size(self.get_folder_size(os.path.join(path, f)))
if os.path.isfile(os.path.join(path, f)):
file_id = fm.get_file_id(path, f)
files[file_id] = (f, self. \
convert_size(self.get_file_size(os.path.join(path, f))))
return directories, files
def start(self, data):
"""This method runs the save_user_data job.
It will received the user file and it will store it in the user space
using the file_manager.
Args:
self: The object pointer.
data: A dictionary containing the user file to be saved.
Returns:
The job_data attribute. The ok attribure will be True if everything
went correctly, False otherwise.
@see file_manager @link avi.utils.data.file_manager.file_manager
"""
from django.core.files.storage import FileSystemStorage
fs = FileSystemStorage()
f = data['file']
file_name = wh().get().USER_FMT % {
"user": "******",
"date": str(round(time.time())),
"name": f.name
}
full_name = os.path.join(wh().get().USER_PATH, file_name)
filename = fs.save(full_name, f)
fm = file_manager()
fm.save_file_info(f.name, full_name, -1, "user", timezone.now())
self.job_data.ok = True
self.job_data.data = {}
self.job_data.data['status'] = 'success'
return self.job_data
def get_file_list(self, path):
"""Deprecated?
Retrieves all the files and directories within a path
Args:
self: The object pointer
path: The path
Returns:
All the files and directories within the given path
"""
"Get a list of folders and files in the current folder"
directory_list = os.listdir(path)
directory_list.sort()
directories = collections.OrderedDict()
files = collections.OrderedDict()
fm = file_manager()
for f in directory_list:
if os.path.isdir(os.path.join(path, f)):
directories[f] = self. \
convert_size(self.get_folder_size(os.path.join(path, f)))
if os.path.isfile(os.path.join(path, f)):
file_id = fm.get_file_id(path, f)
# import re
# files[re.sub('[^A-Za-z0-9._]+.', '', f)] = self. \
files[file_id] = (f, self. \
convert_size(self.get_file_size(os.path.join(path, f))))
#self.log = logger().get_log('directories')
return directories, files
def start(self, data):
"""This method runs the save_samp_data job.
It will received the samp data and it will store it in the user space
using the file_manager.
Args:
self: The object pointer.
data: A dictionary containing the samp data to be saved.
Returns:
The job_data attribute. The ok attribure will be True if everything
went correctly, False otherwise.
@see file_manager @link avi.utils.data.file_manager.file_manager
"""
# FIXME:
fm = file_manager()
file_name = wh().get().USER_FMT % {
"user": "******",
"date": str(round(time.time())),
"name": data['name']
}
fm.save_file_plain_data(unquote(data['data']), file_name,
wh().get().USER_PATH, -1, "user",
timezone.now())
self.job_data.ok = True
self.job_data.data = {}
self.job_data.data['status'] = 'success'
return self.job_data
def delete_file(self, file_name):
"""Deletes the given file
Args:
self: The object pointer
file_name: the file to be deleted
Raises:
Exception: if the an unknow error happens
"""
# file_to_delete = \
# wh_frontend_config().get().CURRENT_PATH + "/" + file_name
path_to_delete = wh_frontend_config().get().CURRENT_PATH
self.log.info(path_to_delete)
self.log.info(file_name)
self.log.info(file_name.startswith('gaia'))
if file_name.startswith('gaia'):
if 'gaia' not in path_to_delete:
path_to_delete = "/data/output/sources/gaia"
elif file_name.startswith('hsa'):
if 'hsa' not in path_to_delete:
path_to_delete = "/data/output/sources/hsa"
elif file_name.startswith('res'):
if 'results' not in path_to_delete:
path_to_delete = "/data/output/results"
elif file_name.startswith('user'):
if 'user' not in path_to_delete:
path_to_delete = "/data/output/user"
try:
# os.remove(file_to_delete)
file_manager().remove_file(file_name, path_to_delete)
# this would be "except OSError, e:" before Python 2.6
except OSError as e:
# errno.ENOENT = no such file or directory
if e.errno != errno.ENOENT:
raise # re-raise exception if a different error occurred
def _get_fits(self, urn, fname=None, save=False):
self.log.info("Getting fits file from urn=%s...", urn)
if not urn:
return None
values = {'PRODUCT.HCSS_URN': urn}
data = urlencode(values)
data = data.encode('utf-8')
#print(self._product_url)
req = Request(self._product_url, data)
response = None
try:
response = urlopen(req)
except url_lib.HTTPError as err:
self.log.warning("Error while retrieving urn info :%i", err.code)
return None
except url_lib.URLError as err:
self.log.warning("Error while retrieving urn info :%i", err.code)
return None
except Exception as err:
self.log.warning("Error while retrieving urn info :%i", err.code)
return None
if not response:
self.log.warning("Invalid response...")
return None
#print(fname)
if fname:
fp = open(fname, 'wb')
fm = file_manager()
fm.save_file_info(fname, fname, self.job_id, "hsa", timezone.now())
else:
fp = tempfile.NamedTemporaryFile(prefix="hsa_fits_",
mode='wb',
delete=False)
fname = fp.name
shutil.copyfileobj(response, fp)
fp.close()
self.log.info("File %s saved", fname)
hdu = fits.open(fname)
return hdu
def get_positional_sources_from_herschel(input_file):
from avi.utils.data.json_manager import json_manager
from avi.utils.data.file_manager import file_manager
from avi.utils.coordinates_manager import coordinates_manager
from avi.core.interface.name_solvers import simbad, ned
jm = json_manager()
fm = file_manager()
cm = coordinates_manager()
r = risea().get()
im = r.interface_manager
data = jm.read_herschel_input(input_file)
c = 0
for i in data:
c += 1
try:
ra = None
dec = None
if i.get('name'):
coords = simbad().get_object_coordinates(i['name'])
if not coords:
coords = ned().get_object_coordinates(i['name'])
if not coords:
continue
v_ra = coords['ra']
v_dec = coords['dec']
else:
v_ra = i.get('ra')
v_dec = i.get('dec')
if not v_ra or not v_dec:
v_l = i.get('l')
v_b = i.get('b')
if not v_l or not v_b:
continue
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
table = None
src = None
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
table = "cat_hppsc_%s" % (str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
table = "cat_spsc_%i" % (wl)
if i['shape'] == 'cone':
radius = float(i['radius'])
src = im._archive_herschel_get_circle(ra, dec, radius, table)
elif i['shape'] == 'box':
width = float(i['width'])
height = float(i['height'])
src = im._archive_herschel_get_box(ra, dec, width, height,
table)
elif i['shape'] == 'polygon':
vertexes = jm.get_vertexes(i)
src = im._archive_herschel_get_polygon(ra, dec, vertexes,
table)
if src != None:
if i.get('output_file'):
fm.save_file_plain_data(src, "%s.vot" % (i['output_file']))
else:
fm.save_file_plain_data(src, "source_%i.vot" % (c))
except ValueError as e:
print("Value Error")
print(traceback.format_exc())
pass
except Exception as e:
print("Exception")
print(traceback.format_exc())
pass
def get_maps_from_herschel(input_file):
from avi.utils.data.json_manager import json_manager
from avi.utils.data.file_manager import file_manager
from avi.utils.coordinates_manager import coordinates_manager
from avi.core.interface.name_solvers import simbad, ned
jm = json_manager()
fm = file_manager()
cm = coordinates_manager()
r = risea().get()
im = r.interface_manager
data = jm.read_herschel_input(input_file)
c = 0
for i in data:
c += 1
try:
ra = None
dec = None
if i.get('name'):
coords = simbad().get_object_coordinates(i['name'])
if not coords:
coords = ned().get_object_coordinates(i['name'])
if not coords:
continue
v_ra = coords['ra']
v_dec = coords['dec']
else:
v_ra = i.get('ra')
v_dec = i.get('dec')
if not v_ra or not v_dec:
v_l = i.get('l')
v_b = i.get('b')
if not v_l or not v_b:
continue
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
table = None
src = None
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
table = "cat_hppsc_%s" % (str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
table = "cat_spsc_%i" % (wl)
radius = float(i['size'])
im.archive_get_maps(ra, dec, radius, 'All', 'PACS', table=table)
except ValueError as e:
print("Value Error")
print(traceback.format_exc())
pass
except Exception as e:
print("Exception")
print(traceback.format_exc())
pass
def get_herschel_data(self, log, data):
"""Does a query to the herschel archive
It will read the input contained in the data parameter and it will
query the herschel archive trough the interface_manager. Then it will
save the results using the file_manager
Args:
self: The object pointer
log: The log
data: The input data to the query
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
file_manager: avi.utils.data.file_manager.file_manager
"""
log.debug('get_herschel_data method')
im = risea().get().interface_manager
fm = file_manager()
cm = coordinates_manager()
jm = json_manager()
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
try:
ra = None
dec = None
if data.get('name') and data.get('name_coord') == 'name':
log.info("Name attr %s found, retrieving coordinates from " \
+ "Simbad/Ned databases", data['name'])
coords = simbad().get_object_coordinates(data['name'])
if not coords:
coords = ned().get_object_coordinates(data['name'])
if not coords:
log.error('Name %s not found in Simbad/Ned data bases',
data['name'])
raise err('Name %s not found in Simbad/Ned data bases',
data['name'])
v_ra = coords['ra']
v_dec = coords['dec']
else:
log.info("Retrieving coordinates from the provided data...")
v_ra = data.get('ra')
v_dec = data.get('dec')
if not v_ra or not v_dec:
log.info("No equatorial coordinates found!")
log.info("Reading galactic coordinates from data...")
v_l = data.get('l')
v_b = data.get('b')
if not v_l or not v_b:
log.error('No valid coordinates found')
raise err('No valid coordinates found')
coords = cm.gal_to_icrs(float(v_l), float(v_b))
ra = coords['ra']
dec = coords['dec']
else:
try:
ra = float(v_ra)
dec = float(v_dec)
except ValueError:
coords = cm.icrs_degrees(v_ra, v_dec)
ra = coords['ra']
dec = coords['dec']
src = None
shape = data['shape']
if shape != 'cone' and shape != 'box' and shape != 'polygon':
log.error("Unknown shape!")
raise err("Unknown shape!")
log.info("Shape: %s", shape)
table = data['table']
if not table or table == "":
table = "v_active_observation"
log.info("Table: %s", "hsa.%s" % (table))
if not data['positional_images']:
log.info("Retrieving positional sources from " +
"the herschel archive...")
if shape == 'cone':
if not data['radius']:
log.error("No radius provided")
raise err("No radius provided")
src = im._archive_herschel_get_circle(
ra, dec, data['radius'], table)
elif shape == 'box':
if not data['width'] or not data['height']:
log.error("No dimensions provided")
raise err("No dimansions provided")
src = im._archive_herschel_get_box(ra, dec, data['width'],
data['height'], table)
elif shape == 'polygon':
vertexes = jm.get_vertexes(data)
src = im._archive_herschel_get_polygon(
ra, dec, vertexes, table)
if src != None:
if not data.get('output_file'):
file_name = wh().get() \
.SOURCES_FMT%{"mission":"hsa",
"date":str(round(time.time())),
"name":"data"}
else:
file_name = wh().get() \
.SOURCES_FMT%{"mission":"hsa",
"date":str(round(time.time())),
"name":data['output_file']}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().HSA_PATH, self.task_id,
"hsa", timezone.now())
#"%f_%f_%s_%s.vot" \
#%(ra,dec,shape,table))
else:
log.error(
"Something went wrong while querying the archive!")
raise err(
"Something went wrong while querying the archive!")
log.info("Everything done!")
return
log.info('Retrieving maps from the herschel archive...')
if shape == 'cone':
if not data['radius']:
log.error("No radius provided")
raise err("No radius provided")
if not data.get('output_file'):
im.archive_get_maps(ra,
dec,
data['radius'],
data['level'],
data['instrument'],
id=self.task_id)
else:
log.info("fileeeeeeeeeee")
im.archive_get_maps(ra,
dec,
data['radius'],
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
elif shape == 'box':
if not data['width'] or not data['height']:
log.error("No dimensions provided")
raise err("No dimansions provided")
if not data.get('output_file'):
im.archive_get_maps_box(ra,
dec,
data['width'],
data['height'],
data['level'],
data['instrument'],
id=self.task_id)
else:
im.archive_get_maps_box(ra,
dec,
data['width'],
data['height'],
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
elif shape == 'polygon':
vertexes = jm.get_vertexes(data)
if not data.get('output_file'):
im.archive_get_maps_polygon(ra,
dec,
vertexes,
data['level'],
data['instrument'],
id=self.task_id)
else:
im.archive_get_maps_polygon(ra,
dec,
vertexes,
data['level'],
data['instrument'],
id=self.task_id,
name=data['output_file'])
log.info("Everything done!")
return
log.error("Something went wrong...")
raise err("Something went wrong...")
except Exception:
log.error(traceback.format_exc())
raise err(traceback.format_exc())
def run(self):
"""Runs the query to the herschel archive.
If the task_data contains the 'input_file' key it will read that value
and call get_herschel_data() once per input parameter found in the
input_file.
If the task_data contains the 'adql' key it will query the archive
through the interface_manager using that query.
Otherwise it will call get_herschel_data() with the input from task_data
Args:
self: The object pointer.
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
get_herschel_data: get_herschel_data()
"""
def get_herschel_data(log, data):
"""Deprecated"""
pass
log = logger().get_log('herschel_query_task')
data = self.task_data.data
jm = json_manager()
if data.get('input_file') and data.get('name_coord') == 'file':
log.info('There is an input file')
#self.get_herschel_data(log, data)
try:
d = jm.read_herschel_input(data['input_file'])
for i in d:
if i.get('name'):
i['name_coord'] = 'name'
if i.get('wavelength'):
wl = int(i['wavelength'])
if wl == 70 or wl == 100 or wl == 160:
i['tablee'] = "hsa.pacs_point_source_%s" % (
str(wl).zfill(3))
#"cat_hppsc_%s"%(str(wl).zfill(3))
elif wl == 250 or wl == 350 or wl == 500:
i['tablee'] = "hsa.spire_point_source_%s" % (wl)
#"cat_spsc_%i"%(wl)
if i.get('positional_source'):
if i['positional_source'] == 'False':
i['positional_images'] = True
else:
i['positional_images'] = False
else:
i['positional_images'] = True
self.get_herschel_data(log, i)
except Exception:
log.error("Exception while retrieving data from herschel")
log.error(traceback.format_exc())
raise err(traceback.format_exc())
finally:
pass
#os.remove(data['input_file'])
return
elif data.get('adql') and data.get('name_coord') == 'adql':
log.info('ADQL query')
im = risea().get().interface_manager
fm = file_manager()
adql = data['adql']
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
src = im._archive_herschel_get_adql(adql)
if src != None:
if not data.get('output_file'):
file_name = wh().get().SOURCES_FMT % {
"mission": "hsa",
"date": str(round(time.time())),
"name": "data"
}
else:
file_name = wh().get().SOURCES_FMT % {
"mission": "hsa",
"date": str(round(time.time())),
"name": data['output_file']
}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().HSA_PATH, self.task_id,
"hsa", timezone.now())
log.info("Everything done!")
return
else:
if data.get('shape') == 'polygon':
jm.set_vertexes(data, data['polygon'])
log.info("added vertexes %s", str(data))
self.get_herschel_data(log, data)
return
def run(self):
"""Runs the query to the gaia archive.
If the task_data contains the 'input_file' key it will read that value
and call get_gaia_data() once per input parameter found in the
input_file.
If the task_data contains the 'adql' key it will query the archive
through the interface_manager using that query.
Otherwise it will call get_gaia_data() with the input from task_data
Args:
self: The object pointer.
Raises:
task_exception: avi.task.task.task_exception
See:
interface_manager: avi.core.interface.interface_manager.interface_manager
See also:
get_gaia_data: get_gaia_data()
"""
log = logger().get_log('gaia_query_task')
jm = json_manager()
data = self.task_data.data
log.info("%s", str(data))
if data.get('input_file') and data.get('name_coord') == 'file':
log.info('There is an input file')
try:
d = jm.read_gaia_input(data['input_file'])
for i in d:
if i.get('name'):
i['name_coord'] = 'name'
self.get_gaia_data(log, i)
except Exception:
log.error("Exception while retrieving data from gaia")
log.error(traceback.format_exc())
raise err(traceback.format_exc())
finally:
os.remove(data['input_file'])
return
elif data.get('adql') and data.get('name_coord') == 'adql':
log.info('ADQL query')
im = risea().get().interface_manager
fm = file_manager()
adql = data['adql']
if not im:
log.error('There is no interface manager initialized!')
raise err("There is no interface manager initialized!")
src = im._archive_gaia_get_adql(adql)
if src != None:
if not data.get('output_file'):
file_name = wh().get().SOURCES_FMT % {
"mission": "gaia",
"date": str(round(time.time())),
"name": "data"
}
else:
file_name = wh().get().SOURCES_FMT % {
"mission": "gaia",
"date": str(round(time.time())),
"name": data['output_file']
}
fm.save_file_plain_data(src, "%s.vot" % (file_name),
wh().get().GAIA_PATH, self.task_id,
"gaia", timezone.now())
log.info("Everything done!")
return
else:
if data.get('shape') == 'polygon':
jm.set_vertexes(data, data['polygon'])
self.get_gaia_data(log, data)
return