class ProcessVOTable(AviTask): """ This function requires a VOTable as an input. Then we present it using pandas. """ sharedfile = AviParameter() outputFile = AviParameter() def output(self): return AviLocalTarget( os.path.join(settings.OUTPUT_PATH, self.outputFile)) def run(self): """ Analyses the VOTable file containing the GACS-dev query results """ shared_file_path = os.path.join(settings.INPUT_PATH, self.sharedfile) logger.error('Input VOTable file: %s' % shared_file_path) t = Table.read(shared_file_path, format='votable') df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames) profile = pandas_profiling.ProfileReport(df) analysis_context = { 'gacs_dfdescription': df.describe().to_html( classes='table table-striped table-bordered table-hover'), 'pandas_profiling': profile.html } # JSON will be the context used for the template with open(self.output().path, 'wb') as out: json.dump(analysis_context, out)
class ProcessData(AviTask): """ This function requires a DownloadData class to be run. We will obtain GACS data in this way. Once we have this data, we parse the VOTable. Then we present it using pandas. """ query = AviParameter() outputFile = AviParameter() def output(self): return AviLocalTarget( os.path.join(settings.OUTPUT_PATH, self.outputFile)) def requires(self): return self.task_dependency(DownloadData) def run(self): """ Analyses the VOTable file containing the GACS-dev query results """ logger.info('Input VOTable file: %s' % self.input().path) t = Table.read(self.input().path, format='votable') df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames) gaiamagcols = [ 'dec', 'dec_error', 'dist', 'phot_g_mean_flux', 'phot_g_mean_mag', 'ra', 'source_id' ] gaiadf = df[gaiamagcols] profile = pandas_profiling.ProfileReport(gaiadf) analysis_context = { 'gacs_dfdescription': gaiadf.describe().to_html( classes='table table-striped table-bordered table-hover'), 'pandas_profiling': profile.html } # logger.debug('analysis_context %s' % analysis_context) # JSON will be the context used for the template with open(self.output().path, 'wb') as out: json.dump(analysis_context, out)
class DownloadData(svc_gacs.GacsQuery): """ This task uses an AVI service, to obtain a data product from GACS. Notice that we do not define a 'run' function! It is defined by the service class which we extend. See :class:`GacsQuery` """ query = AviParameter() outputFile = AviParameter() def output(self): return AviLocalTarget( os.path.join(settings.OUTPUT_PATH, 'simulatedData_%s.vot' % self.outputFile)) def requires(self): return self.task_dependency(DummyTask)
class algorithm(parent): """@class algorithm The algorithm class is the interface to the algorithm_task See: algorithm_task: avi.task.algorithm_task.algorithm_task """ ## Information of the task request request = AviParameter() ## The name of the algorithm alg_name = AviParameter() ## The input parameters params = AviParameter() ## Deprecated results = AviParameter() ## An error message error_message = AviParameter() def output(self): """Deprecated""" algorithm_task().output() return AviLocalTarget("/data/output/test.text") def run(self): """Runs the task Args: self: The object pointer Raises: Exception """ t = algorithm_task() t.task_id = self.request.algorithm_model_model.pk t.task_data.data = self.params try: t.run() except task_exception as err: self.error_message = err.message raise Exception(err.message) except Exception as unknown: self.error_message = unknown.message raise Exception("Unknown error")
class sim_query(parent): """@class herschel_query The herschel_query class is the interface to the herschel_query_task See: herschel_query_task: avi.task.herschel_query_task.herschel_query_task """ ## Information of the task request request = AviParameter() ## name of the object to be queried name = AviParameter() ## Total mass (solar-mass) total_mass = AviParameter() ## virial ratio virial_ratio = AviParameter() ## Half-mass radius (pc) 0.1, 0.5, 1.0 half_mass_radius = AviParameter() ## Fractal dimension fractal_dimension = AviParameter() ## Degree of mass-segregation mass_segregation_degree = AviParameter() ## Binary fraction (%) binary_fraction = AviParameter() def output(self): """Deprecated""" sim_query_task().output() return AviLocalTarget("/data/output/test.txt") def run(self): """Runs the task Args: self: The object pointer Raises: Exception """ log = logger().get_log('risea') log.info('deavi_task run...') t = sim_query_task() t.task_id = self.request.sim_query_model_model.pk data = { 'name': t.task_id, 'total_mass': self.total_mass, 'virial_ratio': self.virial_ratio, 'half_mass_radius': self.half_mass_radius, 'fractal_dimension': self.fractal_dimension, 'mass_segregation_degree': self.mass_segregation_degree, 'binary_fraction': self.binary_fraction } t.task_data.data = data t.run()
class DummyTask(AviTask): """ This is a sample task which has no dependencies. It only exists to further demonstrate dependency creation. """ outputFile = AviParameter() def output(self): return AviLocalTarget( os.path.join(settings.OUTPUT_PATH, 'dummyData_%s.vot' % self.outputFile)) def run(self): time.sleep(3) with open(self.output().path, "w") as outFile: outFile.write("dummyStuff")
class DownloadData(svc_gacs.GacsQuery): """ This task uses an AVI service, to obtain a data product from GACS. Notice that we do not define a 'run' function. It is defined by the service class which we extend. See :class:`GacsQuery` """ query = AviParameter() def output(self): # The AVI job object provides a directory based on a hash of job # parameters which can be used in the AVI # So we create the dir if it doesn't exist already job_dir = self.job_model.request.output_path if not os.path.exists(job_dir): os.makedirs(job_dir) return AviLocalTarget(os.path.join(job_dir, 'gacs_data.vot'))
class ProcessData(AviTask): """ This function requires a DownloadData class to be run. We will obtain Euclid data in this way. Once we have this data, we parse the VOTable. Then we present it using pandas. """ query = AviParameter() def output(self): job_dir = self.job_model.request.output_path return AviLocalTarget(os.path.join(job_dir, "pandas_profile.json")) def requires(self): return self.task_dependency(DownloadData) def run(self): """ Analyses the VOTable file containing the sas-dev query results """ """
class ProcessData(AviTask): """ This function requires a DownloadData class to be run. We will obtain GACS data in this way. Once we have this data, we parse the VOTable. Then we present it using pandas. """ query = AviParameter() def output(self): job_dir = self.job_model.request.output_path return AviLocalTarget(os.path.join(job_dir, "pandas_profile.json")) def requires(self): return self.task_dependency(DownloadData) def run(self): """ Analyses the VOTable file containing the GACS-dev query results """ t = Table.read(self.input().path, format='votable') df = pd.DataFrame(np.ma.filled(t.as_array()), columns=t.colnames) gaiamagcols = [ 'dec', 'dist', 'phot_g_mean_flux', 'phot_g_mean_mag', 'ra', 'source_id' ] gaiadf = df[gaiamagcols] profile = pandas_profiling.ProfileReport(gaiadf) analysis_json_object = {'pandas_profiling': profile.html} panda_str = json.dumps(analysis_json_object) encoded_panda_str = panda_str.encode('utf-8') with open(self.output().path, 'wb') as out: out.write(encoded_panda_str)
class herschel_query(parent): """@class herschel_query The herschel_query class is the interface to the herschel_query_task See: herschel_query_task: avi.task.herschel_query_task.herschel_query_task """ ## Information of the task request request = AviParameter() ## Has the query to be done by coordinates or name? name_coord = AviParameter() ## Name of the object to be queried name = AviParameter() ## Input file containing multiple queries information input_file = AviParameter() ## ra ra = AviParameter() ## dec dec = AviParameter() ## Shape of the query shape = AviParameter() ## Radius of the query radius = AviParameter() ## Width of the query width = AviParameter() ## Height of the query height = AviParameter() ## Array containing the vertexes of a polygon polygon = AviParameter() ## Is it a positional source catalog query or not? positional_images = AviParameter() ## Table of the archive to be queried table = AviParameter() ## Herschel instrument instrument = AviParameter() ## Processing level of the images level = AviParameter() ## Special parameters params = AviParameter() ## Output file name file_name = AviParameter() ## An ADQL query adql = AviParameter() def output(self): """Deprecated""" herschel_query_task().output() return AviLocalTarget("/data/output/test.txt") def run(self): """Runs the task Args: self: The object pointer Raises: Exception """ log = logger().get_log('risea') log.info('deavi_task run...') t = herschel_query_task() t.task_id = self.request.herschel_query_model_model.pk data = { 'name_coord': self.name_coord, 'name': self.name, 'input_file': self.input_file, 'ra': self.ra, 'dec': self.dec, 'shape': self.shape, 'radius': self.radius, 'width': self.width, 'height': self.height, 'polygon': self.polygon, 'positional_images': self.positional_images, 'table': self.table, 'instrument': self.instrument, 'level': self.level, 'params': self.params, 'output_file': self.file_name, 'adql': self.adql } t.task_data.data = data t.run()
class gaia_query(parent): """@class gaia_query The gaia_query class is the interface to the gaia_query_task See: gaia_query_task: avi.task.gaia_query_task.gaia_query_task """ ## Information of the task request request = AviParameter() ## Has the query to be done by coordinates or name? name_coord = AviParameter() ## Name of the object to be queried name = AviParameter() ## Input file containing multiple queries information input_file = AviParameter() ## ra ra = AviParameter() ## dec dec = AviParameter() ## shape of the query shape = AviParameter() ## radius of the query radius = AviParameter() ## width of the query width = AviParameter() ## height of the query height = AviParameter() ## array containing the vertexes of a polygon polygon = AviParameter() ## table of the archive to be queried table = AviParameter() ## special parameters params = AviParameter() ## output file name file_name = AviParameter() ## an ADQL query adql = AviParameter() def output(self): """Deprecated""" gaia_query_task().output() return AviLocalTarget("/data/output/test.txt") def run(self): """Runs the task Args: self: The object pointer Raises: Exception """ t = gaia_query_task() t.task_id = self.request.gaia_query_model_model.pk data = { 'name_coord': self.name_coord, 'name': self.name, 'input_file': self.input_file, 'ra': self.ra, 'dec': self.dec, 'shape': self.shape, 'radius': self.radius, 'width': self.width, 'height': self.height, 'polygon': self.polygon, 'table': self.table, 'params': self.params, 'output_file': self.file_name, 'adql': self.adql } t.task_data.data = data t.run()