class ProcessVOTable(AviTask):
"""
This function requires a VOTable as an input. Then we
present it using pandas.
"""
sharedfile = AviParameter()
outputFile = AviParameter()
def output(self):
return AviLocalTarget(
os.path.join(settings.OUTPUT_PATH, self.outputFile))
def run(self):
"""
Analyses the VOTable file containing the GACS-dev query results
"""
shared_file_path = os.path.join(settings.INPUT_PATH, self.sharedfile)
logger.error('Input VOTable file: %s' % shared_file_path)
t = Table.read(shared_file_path, format='votable')
df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames)
profile = pandas_profiling.ProfileReport(df)
analysis_context = {
'gacs_dfdescription':
df.describe().to_html(
classes='table table-striped table-bordered table-hover'),
'pandas_profiling':
profile.html
}
# JSON will be the context used for the template
with open(self.output().path, 'wb') as out:
json.dump(analysis_context, out)
class ProcessData(AviTask):
"""
This function requires a DownloadData class to be run.
We will obtain GACS data in this way.
Once we have this data, we parse the VOTable. Then we
present it using pandas.
"""
query = AviParameter()
outputFile = AviParameter()
def output(self):
return AviLocalTarget(
os.path.join(settings.OUTPUT_PATH, self.outputFile))
def requires(self):
return self.task_dependency(DownloadData)
def run(self):
"""
Analyses the VOTable file containing the GACS-dev query results
"""
logger.info('Input VOTable file: %s' % self.input().path)
t = Table.read(self.input().path, format='votable')
df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames)
gaiamagcols = [
'dec', 'dec_error', 'dist', 'phot_g_mean_flux', 'phot_g_mean_mag',
'ra', 'source_id'
]
gaiadf = df[gaiamagcols]
profile = pandas_profiling.ProfileReport(gaiadf)
analysis_context = {
'gacs_dfdescription':
gaiadf.describe().to_html(
classes='table table-striped table-bordered table-hover'),
'pandas_profiling':
profile.html
}
# logger.debug('analysis_context %s' % analysis_context)
# JSON will be the context used for the template
with open(self.output().path, 'wb') as out:
json.dump(analysis_context, out)
class DownloadData(svc_gacs.GacsQuery):
"""
This task uses an AVI service, to obtain a data product from GACS.
Notice that we do not define a 'run' function! It is defined by the
service class which we extend.
See :class:`GacsQuery`
"""
query = AviParameter()
outputFile = AviParameter()
def output(self):
return AviLocalTarget(
os.path.join(settings.OUTPUT_PATH,
'simulatedData_%s.vot' % self.outputFile))
def requires(self):
return self.task_dependency(DummyTask)
class algorithm(parent):
"""@class algorithm
The algorithm class is the interface to the algorithm_task
See:
algorithm_task: avi.task.algorithm_task.algorithm_task
"""
## Information of the task request
request = AviParameter()
## The name of the algorithm
alg_name = AviParameter()
## The input parameters
params = AviParameter()
## Deprecated
results = AviParameter()
## An error message
error_message = AviParameter()
def output(self):
"""Deprecated"""
algorithm_task().output()
return AviLocalTarget("/data/output/test.text")
def run(self):
"""Runs the task
Args:
self: The object pointer
Raises:
Exception
"""
t = algorithm_task()
t.task_id = self.request.algorithm_model_model.pk
t.task_data.data = self.params
try:
t.run()
except task_exception as err:
self.error_message = err.message
raise Exception(err.message)
except Exception as unknown:
self.error_message = unknown.message
raise Exception("Unknown error")
class sim_query(parent):
"""@class herschel_query
The herschel_query class is the interface to the herschel_query_task
See:
herschel_query_task: avi.task.herschel_query_task.herschel_query_task
"""
## Information of the task request
request = AviParameter()
## name of the object to be queried
name = AviParameter()
## Total mass (solar-mass)
total_mass = AviParameter()
## virial ratio
virial_ratio = AviParameter()
## Half-mass radius (pc) 0.1, 0.5, 1.0
half_mass_radius = AviParameter()
## Fractal dimension
fractal_dimension = AviParameter()
## Degree of mass-segregation
mass_segregation_degree = AviParameter()
## Binary fraction (%)
binary_fraction = AviParameter()
def output(self):
"""Deprecated"""
sim_query_task().output()
return AviLocalTarget("/data/output/test.txt")
def run(self):
"""Runs the task
Args:
self: The object pointer
Raises:
Exception
"""
log = logger().get_log('risea')
log.info('deavi_task run...')
t = sim_query_task()
t.task_id = self.request.sim_query_model_model.pk
data = {
'name': t.task_id,
'total_mass': self.total_mass,
'virial_ratio': self.virial_ratio,
'half_mass_radius': self.half_mass_radius,
'fractal_dimension': self.fractal_dimension,
'mass_segregation_degree': self.mass_segregation_degree,
'binary_fraction': self.binary_fraction
}
t.task_data.data = data
t.run()
class DummyTask(AviTask):
"""
This is a sample task which has no dependencies. It only exists to further demonstrate dependency creation.
"""
outputFile = AviParameter()
def output(self):
return AviLocalTarget(
os.path.join(settings.OUTPUT_PATH,
'dummyData_%s.vot' % self.outputFile))
def run(self):
time.sleep(3)
with open(self.output().path, "w") as outFile:
outFile.write("dummyStuff")
class DownloadData(svc_gacs.GacsQuery):
"""
This task uses an AVI service, to obtain a data product from GACS.
Notice that we do not define a 'run' function. It is defined by the
service class which we extend.
See :class:`GacsQuery`
"""
query = AviParameter()
def output(self):
# The AVI job object provides a directory based on a hash of job
# parameters which can be used in the AVI
# So we create the dir if it doesn't exist already
job_dir = self.job_model.request.output_path
if not os.path.exists(job_dir):
os.makedirs(job_dir)
return AviLocalTarget(os.path.join(job_dir, 'gacs_data.vot'))
class ProcessData(AviTask):
"""
This function requires a DownloadData class to be run.
We will obtain Euclid data in this way.
Once we have this data, we parse the VOTable. Then we
present it using pandas.
"""
query = AviParameter()
def output(self):
job_dir = self.job_model.request.output_path
return AviLocalTarget(os.path.join(job_dir, "pandas_profile.json"))
def requires(self):
return self.task_dependency(DownloadData)
def run(self):
"""
Analyses the VOTable file containing the sas-dev query results
"""
"""
class ProcessData(AviTask):
"""
This function requires a DownloadData class to be run.
We will obtain GACS data in this way.
Once we have this data, we parse the VOTable. Then we
present it using pandas.
"""
query = AviParameter()
def output(self):
job_dir = self.job_model.request.output_path
return AviLocalTarget(os.path.join(job_dir, "pandas_profile.json"))
def requires(self):
return self.task_dependency(DownloadData)
def run(self):
"""
Analyses the VOTable file containing the GACS-dev query results
"""
t = Table.read(self.input().path, format='votable')
df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames)
gaiamagcols = [
'dec', 'dist', 'phot_g_mean_flux', 'phot_g_mean_mag', 'ra',
'source_id'
]
gaiadf = df[gaiamagcols]
profile = pandas_profiling.ProfileReport(gaiadf)
analysis_json_object = {'pandas_profiling': profile.html}
panda_str = json.dumps(analysis_json_object)
encoded_panda_str = panda_str.encode('utf-8')
with open(self.output().path, 'wb') as out:
out.write(encoded_panda_str)
class herschel_query(parent):
"""@class herschel_query
The herschel_query class is the interface to the herschel_query_task
See:
herschel_query_task: avi.task.herschel_query_task.herschel_query_task
"""
## Information of the task request
request = AviParameter()
## Has the query to be done by coordinates or name?
name_coord = AviParameter()
## Name of the object to be queried
name = AviParameter()
## Input file containing multiple queries information
input_file = AviParameter()
## ra
ra = AviParameter()
## dec
dec = AviParameter()
## Shape of the query
shape = AviParameter()
## Radius of the query
radius = AviParameter()
## Width of the query
width = AviParameter()
## Height of the query
height = AviParameter()
## Array containing the vertexes of a polygon
polygon = AviParameter()
## Is it a positional source catalog query or not?
positional_images = AviParameter()
## Table of the archive to be queried
table = AviParameter()
## Herschel instrument
instrument = AviParameter()
## Processing level of the images
level = AviParameter()
## Special parameters
params = AviParameter()
## Output file name
file_name = AviParameter()
## An ADQL query
adql = AviParameter()
def output(self):
"""Deprecated"""
herschel_query_task().output()
return AviLocalTarget("/data/output/test.txt")
def run(self):
"""Runs the task
Args:
self: The object pointer
Raises:
Exception
"""
log = logger().get_log('risea')
log.info('deavi_task run...')
t = herschel_query_task()
t.task_id = self.request.herschel_query_model_model.pk
data = {
'name_coord': self.name_coord,
'name': self.name,
'input_file': self.input_file,
'ra': self.ra,
'dec': self.dec,
'shape': self.shape,
'radius': self.radius,
'width': self.width,
'height': self.height,
'polygon': self.polygon,
'positional_images': self.positional_images,
'table': self.table,
'instrument': self.instrument,
'level': self.level,
'params': self.params,
'output_file': self.file_name,
'adql': self.adql
}
t.task_data.data = data
t.run()
class gaia_query(parent):
"""@class gaia_query
The gaia_query class is the interface to the gaia_query_task
See:
gaia_query_task: avi.task.gaia_query_task.gaia_query_task
"""
## Information of the task request
request = AviParameter()
## Has the query to be done by coordinates or name?
name_coord = AviParameter()
## Name of the object to be queried
name = AviParameter()
## Input file containing multiple queries information
input_file = AviParameter()
## ra
ra = AviParameter()
## dec
dec = AviParameter()
## shape of the query
shape = AviParameter()
## radius of the query
radius = AviParameter()
## width of the query
width = AviParameter()
## height of the query
height = AviParameter()
## array containing the vertexes of a polygon
polygon = AviParameter()
## table of the archive to be queried
table = AviParameter()
## special parameters
params = AviParameter()
## output file name
file_name = AviParameter()
## an ADQL query
adql = AviParameter()
def output(self):
"""Deprecated"""
gaia_query_task().output()
return AviLocalTarget("/data/output/test.txt")
def run(self):
"""Runs the task
Args:
self: The object pointer
Raises:
Exception
"""
t = gaia_query_task()
t.task_id = self.request.gaia_query_model_model.pk
data = {
'name_coord': self.name_coord,
'name': self.name,
'input_file': self.input_file,
'ra': self.ra,
'dec': self.dec,
'shape': self.shape,
'radius': self.radius,
'width': self.width,
'height': self.height,
'polygon': self.polygon,
'table': self.table,
'params': self.params,
'output_file': self.file_name,
'adql': self.adql
}
t.task_data.data = data
t.run()