def run_pipeline(database, filenames, use_monet=True, profile=False, quiet=False, verbose=False, loglevel='DEBUG'): """ Run a pipeline for a set of parsets. @param database: database name; @param filenames: array of parset filenames; @param use_monet: if True run on MonetDB, otherwise - PostgreSQL; @param profile: use profiler; @param quiet: suppress warnings and log messages; """ def run_parset(pipeline, parname): """ Run single parset @param pipeline: pipeline object to use; @param parname: name of the parset to run """ try: print(parname, end=' ') start = time.time() parset = GSMParset(parname) pipeline.run_parset(parset) print('Time spent: %s seconds' % (time.time() - start)) return True except GSMException as exc: print('ERROR occured: %s' % exc) traceback.print_exc() return False pipeline = GSMPipeline(database=database, use_monet=use_monet, profile=profile) if quiet: switch_console(False) if verbose: switch_console(True) set_all_levels(loglevel.upper()) for parname in filenames: ok = run_parset(pipeline, parname) if not ok: pipeline.reopen_connection()
class SpectraTest(SwitchableTest): def setUp(self): super(SpectraTest, self).setUp() cleanup_db(self.cm.get_connection(database='test')) self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn self.sp = Spectra(self.conn) def tearDown(self): self.conn.commit() def insert_data(self, params, bands=8): self.conn.execute(""" insert into runningcatalog (runcatid, first_xtrsrc_id, datapoints, wm_ra, wm_ra_err, wm_decl, wm_decl_err, x, y, z, healpix_zone) values (100, 1, 1, 1, 0.1, 1, 0.1, 1, 1, 1, 0);""") for band in xrange(1, bands + 1): flux = pow(10, polyval(log10(FREQUENCY[band]), params)) self.conn.execute(""" insert into runningcatalog_fluxes(runcat_id, band, datapoints, wm_f_int, avg_weight_f_int) values(100, %s, 1, %s, 1)""" % (band, flux)) def test_spectra(self): args = [1.0, 0.01, 0.001, 0.0001] self.insert_data(args) fit = self.sp.get_sql_data(100) assert_array_almost_equal(fit, args) def test_spectra2(self): args = [1.0, 0.01, 0.001] self.insert_data(args) fit = self.sp.get_sql_data(100) assert_array_almost_equal(fit, args)
class SpectraTest(SwitchableTest): def setUp(self): super(SpectraTest, self).setUp() cleanup_db(self.cm.get_connection(database='test')) self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn self.sp = Spectra(self.conn) def tearDown(self): self.conn.commit() def insert_data(self, params, bands=8): self.conn.execute(""" insert into runningcatalog (runcatid, first_xtrsrc_id, datapoints, wm_ra, wm_ra_err, wm_decl, wm_decl_err, x, y, z, healpix_zone) values (100, 1, 1, 1, 0.1, 1, 0.1, 1, 1, 1, 0);""") for band in xrange(1, bands+1): flux = pow(10, polyval(log10(FREQUENCY[band]), params)) self.conn.execute(""" insert into runningcatalog_fluxes(runcat_id, band, datapoints, wm_f_int, avg_weight_f_int) values(100, %s, 1, %s, 1)""" % (band, flux)) def test_spectra(self): args = [1.0, 0.01, 0.001, 0.0001] self.insert_data(args) fit = self.sp.get_sql_data(100) assert_array_almost_equal(fit, args) def test_spectra2(self): args = [1.0, 0.01, 0.001] self.insert_data(args) fit = self.sp.get_sql_data(100) assert_array_almost_equal(fit, args)
def setUp(self): super(PipelineGeneralTest, self).setUp() cleanup_db(self.cm.get_connection(database='test')) self.pipeline = GSMPipeline(custom_cm=self.cm, database='test')
@param parname: name of the parset to run """ try: print parname, start = time.time() parset = GSMParset(parname) pipeline.run_parset(parset) print 'Time spent: %s seconds' % (time.time() - start) return True except GSMException, exc: print 'ERROR occured: %s' % exc traceback.print_exc() return False pipeline = GSMPipeline(database=database, use_monet=use_monet, profile=profile) if quiet: switch_console(False) if verbose: switch_console(True) set_all_levels(loglevel.upper()) for parname in filenames: ok = run_parset(pipeline, parname) if not ok: pipeline.reopen_connection() if __name__ == '__main__': parser = argparse.ArgumentParser(description="""
@param parname: name of the parset to run """ try: print parname, start = time.time() parset = GSMParset(parname) pipeline.run_parset(parset) print 'Time spent: %s seconds' % (time.time() - start) return True except GSMException, exc: print 'ERROR occured: %s' % exc traceback.print_exc() return False pipeline = GSMPipeline(database=database, use_monet=use_monet, profile=profile) if quiet: switch_console(False) if verbose: switch_console(True) set_all_levels(loglevel.upper()) for parname in filenames: ok = run_parset(pipeline, parname) if not ok: pipeline.reopen_connection() if __name__ == '__main__':
def setUp(self): self.pipeline = GSMPipeline()
def setUp(self): super(SpectraTest, self).setUp() cleanup_db(self.cm.get_connection(database='test')) self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn self.sp = Spectra(self.conn)