def test_create(self):
dataset = DataSet(data={'description': 'dataset with images'},
database=self.database)
# create 4 images, separated by one day each
image = Image(
dataset=dataset,
data=db_subs.example_dbimage_data_dict())
extracted_source_data = dict(zone=13,
ra=12.12, decl=13.13, ra_err=21.1, decl_err=21.09,
ra_fit_err=1.12, decl_fit_err=1.23,
uncertainty_ew=0.1,uncertainty_ns=0.1,
ew_sys_err=20, ns_sys_err=20,
error_radius=10.0,
x=0.11, y=0.22, z=0.33,
racosdecl=0.44, det_sigma=10.)
src1 = ExtractedSource(data=extracted_source_data, image=image)
src2 = ExtractedSource(data=extracted_source_data, image=image, database=self.database)
self.assertNotEqual(src1.id, src2.id)
extracted_source_data['image'] = image.id
src3 = ExtractedSource(data=extracted_source_data, database=self.database)
extracted_source_data['ra'] = 23.23
src4 = ExtractedSource(data=extracted_source_data, database=self.database, id=src1.id)
self.assertEqual(src1.id, src4.id)
self.assertAlmostEqual(src1.ra, src4.ra)
del extracted_source_data['x']
self.assertRaisesRegexp(
AttributeError, "missing required data key: x",
ExtractedSource, data=extracted_source_data, database=self.database)
def test_infinite(self):
# Check that database insertion doesn't choke on infinite errors.
dataset = DataSet(data={'description': 'example dataset'},
database=self.database)
image = Image(dataset=dataset, data=db_subs.example_dbimage_data_dict())
# Inserting a standard example extractedsource should be fine
extracted_source = db_subs.example_extractedsource_tuple()
image.insert_extracted_sources([extracted_source])
inserted = columns_from_table('extractedsource',
where= {'image' : image.id})
self.assertEqual(len(inserted), 1)
# But if the source has infinite errors we drop it and log a warning
extracted_source = db_subs.example_extractedsource_tuple(error_radius=float('inf'),
peak_err=float('inf'),
flux_err=float('inf'))
# We will add a handler to the root logger which catches all log
# output in a buffer.
iostream = BytesIO()
hdlr = logging.StreamHandler(iostream)
logging.getLogger().addHandler(hdlr)
image.insert_extracted_sources([extracted_source])
logging.getLogger().removeHandler(hdlr)
# We want to be sure that the error has been appropriately logged.
self.assertIn("Dropped source fit with infinite flux errors",
iostream.getvalue())
inserted = columns_from_table('extractedsource',
where= {'image' : image.id})
self.assertEqual(len(inserted), 1)
def test_update(self):
"""
Check that ORM-updates work for the Image class.
"""
dataset1 = DataSet(data={'description':
'dataset with changing images'},
database=self.database)
image_data = db_subs.example_dbimage_data_dict()
image1 = Image(dataset=dataset1, data=image_data)
# Now, update (without changing anything) and make sure it remains the
# same (sanity check):
image1.update()
self.assertEqual(image1.tau_time, image_data['tau_time'])
##This time, change something:
tau_time1 = image_data['tau_time'] + 100
tau_time2 = tau_time1 + 300
tau_time3 = tau_time2 + 300
tau_time4 = tau_time3 + 300
freq_eff1 = image_data['freq_eff']*1.2
image1.update(tau_time = tau_time1)
self.assertEqual(image1.tau_time, tau_time1)
# New 'image' orm-object, created from the database id of image1.
image2 = Image(dataset=dataset1, id=image1.id)
self.assertAlmostEqual(image2.tau_time, tau_time1)
self.assertAlmostEqual(image2.freq_eff, image_data['freq_eff'])
# Same id, so changing image2 changes image1
# but *only* after calling update()
image2.update(tau_time=tau_time2)
self.assertAlmostEqual(image1.tau_time, tau_time1)
image1.update()
self.assertAlmostEqual(image1.tau_time, tau_time2)
image1.update(tau_time=tau_time3)
image2.update()
self.assertAlmostEqual(image2.tau_time, tau_time3)
#Test with multiple items:
image1.update(tau_time=tau_time4, freq_eff=freq_eff1)
self.assertAlmostEqual(image1.tau_time, tau_time4)
self.assertAlmostEqual(image1.freq_eff, freq_eff1)
#And that datetime conversion roundtrips work correctly:
dtime0 = image_data['taustart_ts']
dtime1 = dtime0 + datetime.timedelta(days=3)
self.assertEqual(image1.taustart_ts, dtime0)
self.assertEqual(image2.taustart_ts, dtime0)
image2.update(taustart_ts=dtime1)
#image1 orm-object not yet updated, still caches old value:
self.assertEqual(image1.taustart_ts, dtime0)
self.assertEqual(image2.taustart_ts, dtime1)
image1.update()
self.assertEqual(image1.taustart_ts, dtime1)
def test_update(self):
"""
Check that ORM-updates work for the Image class.
"""
dataset1 = DataSet(
data={'description': 'dataset with changing images'},
database=self.database)
image_data = db_subs.example_dbimage_data_dict()
image1 = Image(dataset=dataset1, data=image_data)
# Now, update (without changing anything) and make sure it remains the
# same (sanity check):
image1.update()
self.assertEqual(image1.tau_time, image_data['tau_time'])
##This time, change something:
tau_time1 = image_data['tau_time'] + 100
tau_time2 = tau_time1 + 300
tau_time3 = tau_time2 + 300
tau_time4 = tau_time3 + 300
freq_eff1 = image_data['freq_eff'] * 1.2
image1.update(tau_time=tau_time1)
self.assertEqual(image1.tau_time, tau_time1)
# New 'image' orm-object, created from the database id of image1.
image2 = Image(dataset=dataset1, id=image1.id)
self.assertAlmostEqual(image2.tau_time, tau_time1)
self.assertAlmostEqual(image2.freq_eff, image_data['freq_eff'])
# Same id, so changing image2 changes image1
# but *only* after calling update()
image2.update(tau_time=tau_time2)
self.assertAlmostEqual(image1.tau_time, tau_time1)
image1.update()
self.assertAlmostEqual(image1.tau_time, tau_time2)
image1.update(tau_time=tau_time3)
image2.update()
self.assertAlmostEqual(image2.tau_time, tau_time3)
#Test with multiple items:
image1.update(tau_time=tau_time4, freq_eff=freq_eff1)
self.assertAlmostEqual(image1.tau_time, tau_time4)
self.assertAlmostEqual(image1.freq_eff, freq_eff1)
#And that datetime conversion roundtrips work correctly:
dtime0 = image_data['taustart_ts']
dtime1 = dtime0 + datetime.timedelta(days=3)
self.assertEqual(image1.taustart_ts, dtime0)
self.assertEqual(image2.taustart_ts, dtime0)
image2.update(taustart_ts=dtime1)
#image1 orm-object not yet updated, still caches old value:
self.assertEqual(image1.taustart_ts, dtime0)
self.assertEqual(image2.taustart_ts, dtime1)
image1.update()
self.assertEqual(image1.taustart_ts, dtime1)
def test_create2(self):
dataset1 = DataSet(data={'description': 'dataset with images'},
database=self.database)
self.assertEqual(dataset1.images, set())
image_data = db_subs.example_dbimage_data_dict()
image1 = Image(dataset=dataset1, data=image_data)
image2 = Image(dataset=dataset1, data=image_data)
extractedsource_data = {
'ra': 123.123,
'decl': 23.23,
'ra_err': 21.1,
'decl_err': 21.09,
'ra_fit_err': 0.1,
'decl_fit_err': 0.1,
'uncertainty_ew': 0.1,
'uncertainty_ns': 0.1,
'zone': 1,
'x': 0.11,
'y': 0.22,
'z': 0.33,
'racosdecl': 0.44,
'det_sigma': 10.0,
'ew_sys_err': 20,
'ns_sys_err': 20,
'error_radius': 10.0
}
source1 = ExtractedSource(image=image1, data=extractedsource_data)
extractedsource_data['ra'] = 45.45
extractedsource_data['decl'] = 55.55
source2 = ExtractedSource(image=image1, data=extractedsource_data)
self.assertEqual(len(image1.sources), 2)
# Source #3 points to the same source as #2
source3 = ExtractedSource(id=source2.id, database=self.database)
# Which means there are no extra sources for image1
self.assertEqual(len(image1.sources), 2)
srcids = set([source.id for source in image1.sources])
# If, however, we create a new source with
# an image reference in the constructor, we get a
# 'deep' copy:
source4 = ExtractedSource(image=image1, id=source2.id)
# Now there's a new source for image1!
self.assertEqual(len(image1.sources), 3)
# But if we filter on the source ids,
# we see there are really only two sources
srcids = set([source.id for source in image1.sources])
self.assertEqual(set([source1.id, source2.id]), srcids)
extractedsource_data['ra'] = 89.89
extractedsource_data['decl'] = 78.78
source5 = ExtractedSource(image=image2, data=extractedsource_data)
self.assertEqual(len(image2.sources), 1)
self.assertEqual(image2.sources.pop().id, source5.id)
def test_update(self):
image_data = db_subs.example_dbimage_data_dict()
dataset1 = DataSet(data={'description': 'dataset with images'},
database=self.database)
self.assertEqual(dataset1.images, set())
image1 = Image(dataset=dataset1, data=image_data)
image2 = Image(dataset=dataset1, data=image_data)
extractedsource_data = {
'ra': 123.123,
'decl': 23.23,
'ra_err': 21.1,
'decl_err': 21.09,
'ra_fit_err': 0.1,
'decl_fit_err': 0.1,
'uncertainty_ew': 0.1,
'uncertainty_ns': 0.1,
'zone': 1,
'x': 0.11,
'y': 0.22,
'z': 0.33,
'racosdecl': 0.44,
'det_sigma': 11.1,
'ew_sys_err': 20,
'ns_sys_err': 20,
'error_radius': 10.0
}
source1 = ExtractedSource(image=image1, data=extractedsource_data)
extractedsource_data['ra'] = 45.45
extractedsource_data['decl'] = 55.55
source2 = ExtractedSource(image=image1, data=extractedsource_data)
self.assertEqual(len(image1.sources), 2)
# Source #3 points to the same source as #2
source3 = ExtractedSource(id=source2.id, database=self.database)
# Update source3
source3.update(decl=44.44)
# But no change for #1 and #2
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 55.55)
self.assertAlmostEqual(source3.decl, 44.44)
source1.update() # nothing changes for #1
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 55.55)
self.assertAlmostEqual(source3.decl, 44.44)
# Now we make sure source #2 takes note of the change done through #3
source2.update()
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 44.44)
def test_create(self):
dataset = DataSet(data={'description': 'dataset with images'},
database=self.database)
# create 4 images, separated by one day each
image = Image(dataset=dataset,
data=db_subs.example_dbimage_data_dict())
extracted_source_data = dict(zone=13,
ra=12.12,
decl=13.13,
ra_err=21.1,
decl_err=21.09,
ra_fit_err=1.12,
decl_fit_err=1.23,
uncertainty_ew=0.1,
uncertainty_ns=0.1,
ew_sys_err=20,
ns_sys_err=20,
error_radius=10.0,
x=0.11,
y=0.22,
z=0.33,
racosdecl=0.44,
det_sigma=10.)
src1 = ExtractedSource(data=extracted_source_data, image=image)
src2 = ExtractedSource(data=extracted_source_data,
image=image,
database=self.database)
self.assertNotEqual(src1.id, src2.id)
extracted_source_data['image'] = image.id
src3 = ExtractedSource(data=extracted_source_data,
database=self.database)
extracted_source_data['ra'] = 23.23
src4 = ExtractedSource(data=extracted_source_data,
database=self.database,
id=src1.id)
self.assertEqual(src1.id, src4.id)
self.assertAlmostEqual(src1.ra, src4.ra)
del extracted_source_data['x']
self.assertRaisesRegexp(AttributeError,
"missing required data key: x",
ExtractedSource,
data=extracted_source_data,
database=self.database)
def test_create2(self):
dataset1 = DataSet(data={'description': 'dataset with images'},
database=self.database)
self.assertEqual(dataset1.images, set())
image_data = db_subs.example_dbimage_data_dict()
image1 = Image(dataset=dataset1, data=image_data)
image2 = Image(dataset=dataset1, data=image_data)
extractedsource_data = {'ra': 123.123, 'decl': 23.23,
'ra_err': 21.1, 'decl_err': 21.09,
'ra_fit_err': 0.1, 'decl_fit_err': 0.1,
'uncertainty_ew': 0.1, 'uncertainty_ns': 0.1,
'zone': 1, 'x': 0.11, 'y': 0.22, 'z': 0.33,
'racosdecl': 0.44,
'det_sigma': 10.0,
'ew_sys_err': 20, 'ns_sys_err': 20,
'error_radius': 10.0}
source1 = ExtractedSource(image=image1, data=extractedsource_data)
extractedsource_data['ra'] = 45.45
extractedsource_data['decl'] = 55.55
source2 = ExtractedSource(
image=image1, data=extractedsource_data)
self.assertEqual(len(image1.sources), 2)
# Source #3 points to the same source as #2
source3 = ExtractedSource(id=source2.id, database=self.database)
# Which means there are no extra sources for image1
self.assertEqual(len(image1.sources), 2)
srcids = set([source.id for source in image1.sources])
# If, however, we create a new source with
# an image reference in the constructor, we get a
# 'deep' copy:
source4 = ExtractedSource(image=image1, id=source2.id)
# Now there's a new source for image1!
self.assertEqual(len(image1.sources), 3)
# But if we filter on the source ids,
# we see there are really only two sources
srcids = set([source.id for source in image1.sources])
self.assertEqual(set([source1.id, source2.id]), srcids)
extractedsource_data['ra'] = 89.89
extractedsource_data['decl'] = 78.78
source5 = ExtractedSource(image=image2, data=extractedsource_data)
self.assertEqual(len(image2.sources), 1)
self.assertEqual(image2.sources.pop().id, source5.id)
def test_update(self):
image_data = db_subs.example_dbimage_data_dict()
dataset1 = DataSet(data={'description': 'dataset with images'},
database=self.database)
self.assertEqual(dataset1.images, set())
image1 = Image(dataset=dataset1, data=image_data)
image2 = Image(dataset=dataset1, data=image_data)
extractedsource_data = {'ra': 123.123, 'decl': 23.23,
'ra_err': 21.1, 'decl_err': 21.09,
'ra_fit_err': 0.1, 'decl_fit_err': 0.1,
'uncertainty_ew': 0.1, 'uncertainty_ns': 0.1,
'zone': 1, 'x': 0.11, 'y': 0.22, 'z': 0.33,
'racosdecl': 0.44,
'det_sigma': 11.1,
'ew_sys_err': 20, 'ns_sys_err': 20,
'error_radius': 10.0}
source1 = ExtractedSource(image=image1, data=extractedsource_data)
extractedsource_data['ra'] = 45.45
extractedsource_data['decl'] = 55.55
source2 = ExtractedSource(image=image1, data=extractedsource_data)
self.assertEqual(len(image1.sources), 2)
# Source #3 points to the same source as #2
source3 = ExtractedSource(id=source2.id, database=self.database)
# Update source3
source3.update(decl=44.44)
# But no change for #1 and #2
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 55.55)
self.assertAlmostEqual(source3.decl, 44.44)
source1.update() # nothing changes for #1
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 55.55)
self.assertAlmostEqual(source3.decl, 44.44)
# Now we make sure source #2 takes note of the change done through #3
source2.update()
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source1.decl, 23.23)
self.assertAlmostEqual(source2.decl, 44.44)
def test_infinite(self):
# Check that database insertion doesn't choke on infinite errors.
dataset = DataSet(data={'description': 'example dataset'},
database=self.database)
image = Image(dataset=dataset,
data=db_subs.example_dbimage_data_dict())
# Inserting a standard example extractedsource should be fine
extracted_source = db_subs.example_extractedsource_tuple()
image.insert_extracted_sources([extracted_source])
inserted = columns_from_table('extractedsource',
where={'image': image.id})
self.assertEqual(len(inserted), 1)
# But if the source has infinite errors we drop it and log a warning
extracted_source = db_subs.example_extractedsource_tuple(
error_radius=float('inf'),
peak_err=float('inf'),
flux_err=float('inf'))
# We will add a handler to the root logger which catches all log
# output in a buffer.
iostream = BytesIO()
hdlr = logging.StreamHandler(iostream)
logging.getLogger().addHandler(hdlr)
image.insert_extracted_sources([extracted_source])
logging.getLogger().removeHandler(hdlr)
# We want to be sure that the error has been appropriately logged.
self.assertIn("Dropped source fit with infinite flux errors",
iostream.getvalue())
inserted = columns_from_table('extractedsource',
where={'image': image.id})
self.assertEqual(len(inserted), 1)
def setUp(self):
import tkp.db.database
self.database = tkp.db.database.Database()
# Basic template data for each image.
self.image_data = db_subs.example_dbimage_data_dict()
def setUp(self):
import tkp.db.database
self.database = tkp.db.database.Database()
# Basic template data for each image.
self.image_data = db_subs.example_dbimage_data_dict()
def test_lightsurface(self):
images = []
# make 4 * 5 images with different frequencies and date
for frequency in [80e6, 90e6, 100e6, 110e6, 120e6]:
for day in [3, 4, 5, 6]:
img_data = db_subs.example_dbimage_data_dict(
taustart_ts=datetime.datetime(2010, 3, day),
freq_eff=frequency)
image = Image(dataset=self.dataset, data=img_data)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.111 + i,
'decl': 11.11 + i,
'ra_fit_err': 0.01,
'decl_fit_err': 0.01,
'i_peak': 10. * i,
'i_peak_err': 0.1,
'error_radius': 10.0,
'fit_type': 1,
# x=0.11, y=0.22, z=0.33, det_sigma=11.1, zone=i
})
# Insert the 3 sources in each image, while further varying the flux
for i, image in enumerate(images):
# Create the "source finding results"
sources = []
for data in data_list:
source = db_subs.example_extractedsource_tuple(
ra=data['ra'],
dec=data['decl'],
ra_fit_err=data['ra_fit_err'],
dec_fit_err=data['decl_fit_err'],
peak=data['i_peak'] * (1 + i),
peak_err=data['i_peak_err'],
flux=data['i_peak'] * (1 + i),
flux_err=data['i_peak_err'],
fit_type=data['fit_type'])
sources.append(source)
# Insert the sources
insert_extracted_sources(image._id, sources)
# Run the association for each list of source for an image
associate_extracted_sources(image._id,
deRuiter_r=3.7,
new_source_sigma_margin=3)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick any image, select the first source (smallest RA)
# and extract its light curve
# TODO: aaarch this is so ugly. Because this a set we need to pop it.
sources = self.dataset.images.pop().sources
#sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
extracted_source = sources[0].id
lightcurve = tkp.db.general.lightcurve(extracted_source)
def test_lightsurface(self):
images = []
# make 4 * 5 images with different frequencies and date
for frequency in [80e6, 90e6, 100e6, 110e6, 120e6]:
for day in [3, 4, 5, 6]:
img_data = db_subs.example_dbimage_data_dict(
taustart_ts=datetime.datetime(2010, 3, day),
freq_eff = frequency
)
image = Image(dataset=self.dataset, data=img_data)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.111 + i,
'decl': 11.11 + i,
'ra_fit_err': 0.01,
'decl_fit_err': 0.01,
'i_peak': 10.*i,
'i_peak_err': 0.1,
'error_radius': 10.0,
'fit_type': 1,
# x=0.11, y=0.22, z=0.33, det_sigma=11.1, zone=i
})
# Insert the 3 sources in each image, while further varying the flux
for i, image in enumerate(images):
# Create the "source finding results"
sources = []
for data in data_list:
source = db_subs.example_extractedsource_tuple(
ra=data['ra'], dec=data['decl'],
ra_fit_err=data['ra_fit_err'],
dec_fit_err= data['decl_fit_err'],
peak = data['i_peak']*(1+i),
peak_err = data['i_peak_err'],
flux = data['i_peak']*(1+i),
flux_err = data['i_peak_err'],
fit_type=data['fit_type']
)
sources.append(source)
# Insert the sources
insert_extracted_sources(image._id, sources)
# Run the association for each list of source for an image
associate_extracted_sources(image._id, deRuiter_r=3.7,
new_source_sigma_margin=3)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick any image, select the first source (smallest RA)
# and extract its light curve
# TODO: aaarch this is so ugly. Because this a set we need to pop it.
sources = self.dataset.images.pop().sources
#sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
extracted_source = sources[0].id
lightcurve = tkp.db.general.lightcurve(extracted_source)
