def run_pipeline(database,
filenames,
use_monet=True,
profile=False,
quiet=False,
verbose=False,
loglevel='DEBUG'):
"""
Run a pipeline for a set of parsets.
@param database: database name;
@param filenames: array of parset filenames;
@param use_monet: if True run on MonetDB, otherwise - PostgreSQL;
@param profile: use profiler;
@param quiet: suppress warnings and log messages;
"""
def run_parset(pipeline, parname):
"""
Run single parset
@param pipeline: pipeline object to use;
@param parname: name of the parset to run
"""
try:
print(parname, end=' ')
start = time.time()
parset = GSMParset(parname)
pipeline.run_parset(parset)
print('Time spent: %s seconds' % (time.time() - start))
return True
except GSMException as exc:
print('ERROR occured: %s' % exc)
traceback.print_exc()
return False
pipeline = GSMPipeline(database=database,
use_monet=use_monet,
profile=profile)
if quiet:
switch_console(False)
if verbose:
switch_console(True)
set_all_levels(loglevel.upper())
for parname in filenames:
ok = run_parset(pipeline, parname)
if not ok:
pipeline.reopen_connection()
class SpectraTest(SwitchableTest):
def setUp(self):
super(SpectraTest, self).setUp()
cleanup_db(self.cm.get_connection(database='test'))
self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn
self.sp = Spectra(self.conn)
def tearDown(self):
self.conn.commit()
def insert_data(self, params, bands=8):
self.conn.execute("""
insert into runningcatalog (runcatid, first_xtrsrc_id, datapoints,
wm_ra, wm_ra_err, wm_decl, wm_decl_err, x, y, z, healpix_zone)
values (100, 1, 1, 1, 0.1, 1, 0.1, 1, 1, 1, 0);""")
for band in xrange(1, bands + 1):
flux = pow(10, polyval(log10(FREQUENCY[band]), params))
self.conn.execute("""
insert into runningcatalog_fluxes(runcat_id, band, datapoints, wm_f_int, avg_weight_f_int)
values(100, %s, 1, %s, 1)""" % (band, flux))
def test_spectra(self):
args = [1.0, 0.01, 0.001, 0.0001]
self.insert_data(args)
fit = self.sp.get_sql_data(100)
assert_array_almost_equal(fit, args)
def test_spectra2(self):
args = [1.0, 0.01, 0.001]
self.insert_data(args)
fit = self.sp.get_sql_data(100)
assert_array_almost_equal(fit, args)
class SpectraTest(SwitchableTest):
def setUp(self):
super(SpectraTest, self).setUp()
cleanup_db(self.cm.get_connection(database='test'))
self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn
self.sp = Spectra(self.conn)
def tearDown(self):
self.conn.commit()
def insert_data(self, params, bands=8):
self.conn.execute("""
insert into runningcatalog (runcatid, first_xtrsrc_id, datapoints,
wm_ra, wm_ra_err, wm_decl, wm_decl_err, x, y, z, healpix_zone)
values (100, 1, 1, 1, 0.1, 1, 0.1, 1, 1, 1, 0);""")
for band in xrange(1, bands+1):
flux = pow(10, polyval(log10(FREQUENCY[band]), params))
self.conn.execute("""
insert into runningcatalog_fluxes(runcat_id, band, datapoints, wm_f_int, avg_weight_f_int)
values(100, %s, 1, %s, 1)""" % (band, flux))
def test_spectra(self):
args = [1.0, 0.01, 0.001, 0.0001]
self.insert_data(args)
fit = self.sp.get_sql_data(100)
assert_array_almost_equal(fit, args)
def test_spectra2(self):
args = [1.0, 0.01, 0.001]
self.insert_data(args)
fit = self.sp.get_sql_data(100)
assert_array_almost_equal(fit, args)
def setUp(self):
super(PipelineGeneralTest, self).setUp()
cleanup_db(self.cm.get_connection(database='test'))
self.pipeline = GSMPipeline(custom_cm=self.cm, database='test')
@param parname: name of the parset to run
"""
try:
print parname,
start = time.time()
parset = GSMParset(parname)
pipeline.run_parset(parset)
print 'Time spent: %s seconds' % (time.time() - start)
return True
except GSMException, exc:
print 'ERROR occured: %s' % exc
traceback.print_exc()
return False
pipeline = GSMPipeline(database=database,
use_monet=use_monet,
profile=profile)
if quiet:
switch_console(False)
if verbose:
switch_console(True)
set_all_levels(loglevel.upper())
for parname in filenames:
ok = run_parset(pipeline, parname)
if not ok:
pipeline.reopen_connection()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="""
@param parname: name of the parset to run
"""
try:
print parname,
start = time.time()
parset = GSMParset(parname)
pipeline.run_parset(parset)
print 'Time spent: %s seconds' % (time.time() - start)
return True
except GSMException, exc:
print 'ERROR occured: %s' % exc
traceback.print_exc()
return False
pipeline = GSMPipeline(database=database,
use_monet=use_monet,
profile=profile)
if quiet:
switch_console(False)
if verbose:
switch_console(True)
set_all_levels(loglevel.upper())
for parname in filenames:
ok = run_parset(pipeline, parname)
if not ok:
pipeline.reopen_connection()
if __name__ == '__main__':
def setUp(self):
self.pipeline = GSMPipeline()
def setUp(self):
super(SpectraTest, self).setUp()
cleanup_db(self.cm.get_connection(database='test'))
self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn
self.sp = Spectra(self.conn)
def setUp(self):
super(SpectraTest, self).setUp()
cleanup_db(self.cm.get_connection(database='test'))
self.conn = GSMPipeline(custom_cm=self.cm, database='test').conn
self.sp = Spectra(self.conn)