def testUpdate(self):
print 'Testing PySourceCatalog: updating a source'
s1 = cp.PySource()
s1.setParameter('speed', 12.345, 0.987, 'm/s')
s1.ID = 0
s2 = s1.copy()
sc = cp.PySourceCatalog()
sc.insert(s1)
sc.update(s1.ID, s2)
def testUpdateSourcesDuplicates(self):
print 'Testing PySourceCatalog: updating source dictionary, test for assert on duplicate'
s1 = cp.PySource()
s1.setParameter('speed1', 12.345, 0.987, 'm/s')
s1.ID = 0
sc = cp.PySourceCatalog()
sc.setSources({s1.ID: s1})
sc.updateSources({s1.ID: s1}, warn_on_duplicate=False) # this should silently overwrite
with self.assertRaises(AssertionError):
sc.updateSources({s1.ID: s1})
def testUpdateSources(self):
print 'Testing PySourceCatalog: updating source dictionary'
s1 = cp.PySource()
s1.setParameter('speed1', 12.345, 0.987, 'm/s')
s1.ID = 0
s2 = s1.copy()
s2.ID = 1
sc = cp.PySourceCatalog()
sc.setSources({s1.ID: s1})
sc.updateSources({s2.ID: s2})
def testDoubleInsert(self):
print 'Testing PySourceCatalog: exception on inserting a source with same ID'
s1 = cp.PySource()
s1.setParameter('speed', 12.345, 0.987, 'm/s')
s1.ID = 0
s2 = s1.copy()
sc = cp.PySourceCatalog()
sc.insert(s1)
with self.assertRaises(ValueError):
sc.insert(s2)
def testUpdateNotExistent(self):
print 'Testing PySourceCatalog: exception on updating a source with missing ID'
s1 = cp.PySource()
s1.setParameter('speed', 12.345, 0.987, 'm/s')
s1.ID = 0
s2 = s1.copy()
s2.ID = 1000
sc = cp.PySourceCatalog()
sc.insert(s1)
with self.assertRaises(ValueError):
sc.update(1000, s2)
def testSetSources(self):
print 'Testing PySourceCatalog: inserting source dictionary'
s1 = cp.PySource()
s2 = cp.PySource()
s1.ID = 10 # this is important, otherwise sources get overwritten
s2.ID = 11
s1.setParameter('speed1', 12.345, 0.987, 'm/s')
s2.setParameter('speed2', 12.345, 0.987, 'm/s')
sDict = {s1.ID: s1, s2.ID: s2}
sc = cp.PySourceCatalog()
sc.setSources(sDict)
def testUpdateIDmismatch(self):
print 'Testing PySourceCatalog: updating a source, mismatch exception on source id and call'
s1 = cp.PySource()
s1.setParameter('speed', 12.345, 0.987, 'm/s')
s1.ID = 0
s2 = s1.copy()
self.assertEqual(s1.ID, s2.ID)
s2.ID = 10
sc = cp.PySourceCatalog()
sc.insert(s1)
with self.assertRaises(AssertionError):
sc.update(0, s2)
def testGetSourceIDs(self):
print 'Testing PySourceCatalog: getting source IDs'
s1 = cp.PySource()
s2 = cp.PySource()
s1.ID = 10 # this is important, otherwise sources get overwritten
s2.ID = 11
s1.setParameter('speed1', 12.345, 0.987, 'm/s')
s2.setParameter('speed2', 12.345, 0.987, 'm/s')
sDict = {s1.ID: s1, s2.ID: s2}
sc = cp.PySourceCatalog()
sc.setSources(sDict)
allIDs = sc.getSourceIDs()
self.assertIsInstance(allIDs, list)
self.assertEqual(allIDs, [10, 11])
def testRunTypeSafety(self): print 'Testing PyModuleParametrisation: run method type safety' p = cp.PyModuleParametrisation() cube = np.zeros((100, 100, 100), dtype=np.float32) mask = np.zeros((100, 100, 100), dtype=np.int16) initcatalog = cp.PySourceCatalog() doMaskOptimization = True doBusyFitting = True p.setFlags(doMaskOptimization, doBusyFitting) with self.assertRaises(ValueError): p.run(cube.astype(np.float64), mask, initcatalog) with self.assertRaises(ValueError): p.run(cube, mask.astype(np.float64), initcatalog) with self.assertRaises(TypeError): p.run(cube, mask, 1)
def testGetSources(self):
print 'Testing PySourceCatalog: getting source dictionary'
s1 = cp.PySource()
s2 = cp.PySource()
s1.ID = 10 # this is important, otherwise sources get overwritten
s2.ID = 11
s1.setParameter('speed1', 12.345, 0.987, 'm/s')
s2.setParameter('speed2', 12.345, 0.987, 'm/s')
sDict = {s1.ID: s1, s2.ID: s2}
sc = cp.PySourceCatalog()
sc.setSources(sDict)
sDictReturned = sc.getSources()
self.assertIsInstance(sDictReturned, dict)
self.assertEqual(len(sDict), len(sDictReturned))
self.assertIsInstance(sDictReturned.values()[0], cp.PySource)
def testSetSourcesTypeSafety(self): print 'Testing PySourceCatalog: inserting source dictionary, type safety' with self.assertRaises(TypeError): sc = cp.PySourceCatalog() sc.setSources(1)
def testConstructorTypeSafety(self): print 'Testing PySourceCatalog: constructor type safety' with self.assertRaises(TypeError): sc = cp.PySourceCatalog(2)
def testInsertTypeSafety(self): print 'Testing PySourceCatalog: inserting a source, type safety' with self.assertRaises(TypeError): sc = cp.PySourceCatalog() sc.insert(1)
def testInsert(self):
print 'Testing PySourceCatalog: inserting a source'
s = cp.PySource()
s.setParameter('speed', 12.345, 0.987, 'm/s')
sc = cp.PySourceCatalog()
sc.insert(s)
def testRun(self):
print 'Testing PyModuleParametrisation: run method (artificial data)'
def addGauss(sName, sID, cube, mask, amp, x0, y0, z0, xw, yw, zw):
"""adds a fake source (3D gauss) and initial mask (few pixels large)
since cube/mask are ndarrays, it's 'call-by-reference'
returns a PySource with just the right parameters"""
def gaussian1D(dist, sigma):
return np.exp(- dist**2 / 2.0 / sigma**2)
# first need xyz indices
z, y, x = np.indices(cube.shape)
cube += amp * gaussian1D(x-x0, xw) * gaussian1D(y-y0, yw) * gaussian1D(z-z0, zw)
mask[
z0 - 3*zw: z0 + 3*zw,
y0 - 3*yw: y0 + 3*yw,
x0 - 3*xw: x0 + 3*xw,
] = sID
s = cp.PySource()
s.name = sName
s.ID = sID
pDict = {
'X': cp.PyMeasurement('X', x0, 0.0, ''),
'Y': cp.PyMeasurement('Y', y0, 0.0, ''),
'Z': cp.PyMeasurement('Z', z0, 0.0, ''),
'BBOX_X_MIN': cp.PyMeasurement('BBOX_X_MIN', x0 - 3*xw, 0.0, ''),
'BBOX_X_MAX': cp.PyMeasurement('BBOX_X_MAX', x0 + 3*xw, 0.0, ''),
'BBOX_Y_MIN': cp.PyMeasurement('BBOX_Y_MIN', y0 - 3*yw, 0.0, ''),
'BBOX_Y_MAX': cp.PyMeasurement('BBOX_Y_MAX', y0 + 3*yw, 0.0, ''),
'BBOX_Z_MIN': cp.PyMeasurement('BBOX_Z_MIN', z0 - 3*zw, 0.0, ''),
'BBOX_Z_MAX': cp.PyMeasurement('BBOX_Z_MAX', z0 + 3*zw, 0.0, ''),
}
s.setParameters(pDict)
return s
cube = np.random.normal(0.0, 0.01, (100, 100, 100)).astype(np.float32)
mask = np.zeros((100, 100, 100), dtype=np.int16)
# add some sources
initcatalog = cp.PySourceCatalog()
initcatalog.insert(addGauss("source1", 1, cube, mask, 10.0, 20, 25, 30, 2, 3, 5))
initcatalog.insert(addGauss("source2", 2, cube, mask, 20.0, 50, 55, 60, 5, 3, 2))
for k, v in initcatalog.getSources().iteritems():
print '\n-----------------'
print 'source', k
print '-----------------'
for _k, _v in sorted(v.getParameters().iteritems()):
print _k, _v.asString()
print "\nmax amp"
print np.where(cube == np.max(cube)), np.max(cube)
p = cp.PyModuleParametrisation()
doMaskOptimization = True
doBusyFitting = False
p.setFlags(doMaskOptimization, doBusyFitting)
p.run(cube, mask, initcatalog)
results = p.getCatalog()
for k, v in results.getSources().iteritems():
print '\n-----------------'
print 'source', k
print '-----------------'
for _k, _v in sorted(v.getParameters().iteritems()):
print _k, _v.asString()
self.assertIsInstance(results.getSources(), dict)
self.assertEqual(len(results.getSources()), 2)
s1params = results.getSources()[1].getParameters()
s2params = results.getSources()[2].getParameters()
self.assertAlmostEqual(s1params['RMS_CUBE'].value, 0.01, 1)
self.assertAlmostEqual(s1params['F_PEAK'].value, 10.0, 1)
self.assertAlmostEqual(s1params['X'].value, 20.0, 1)
self.assertAlmostEqual(s1params['Y'].value, 25.0, 1)
self.assertAlmostEqual(s1params['Z'].value, 30.0, 1)
self.assertAlmostEqual(s2params['F_PEAK'].value, 20.0, 1)
self.assertAlmostEqual(s2params['X'].value, 50.0, 1)
self.assertAlmostEqual(s2params['Y'].value, 55.0, 1)
self.assertAlmostEqual(s2params['Z'].value, 60.0, 1)
from sofia import store_ascii
store_ascii.make_ascii(results, ['*'], outname='/tmp/sofia_catalog_test1.txt')
store_ascii.make_ascii(results, ['X', 'Y', 'Z'], outname='/tmp/sofia_catalog_test2.txt')
from sofia import store_xml
store_xml.make_xml(results, outname='/tmp/sofia_catalog_test1.xml')
def parametrise(cube, mask, objects, cathead, catformt, catparunits,
Parameters, dunits):
cathead = np.array(cathead)
objects = np.array(objects)
initcatalog = cp.PySourceCatalog()
for obj in objects:
# Check flags
source_flag = create_source_flags(
cube, mask, cathead, obj[cathead == "id"], obj[cathead == "x_min"],
obj[cathead == "x_max"], obj[cathead == "y_min"],
obj[cathead == "y_max"], obj[cathead == "z_min"],
obj[cathead == "z_max"])
newSource = cp.PySource()
newSource.ID = obj[cathead == "id"]
newParamsDict = {
"flag": cp.PyMeasurement("flag", source_flag, 0.0, ""),
"x": cp.PyMeasurement("x", obj[cathead == "x_geo"], 0.0, ""),
"y": cp.PyMeasurement("y", obj[cathead == "y_geo"], 0.0, ""),
"z": cp.PyMeasurement("z", obj[cathead == "z_geo"], 0.0, ""),
"x_min": cp.PyMeasurement("x_min", obj[cathead == "x_min"], 0.0,
""),
"x_max": cp.PyMeasurement("x_max", obj[cathead == "x_max"], 0.0,
""),
"y_min": cp.PyMeasurement("y_min", obj[cathead == "y_min"], 0.0,
""),
"y_max": cp.PyMeasurement("y_max", obj[cathead == "y_max"], 0.0,
""),
"z_min": cp.PyMeasurement("z_min", obj[cathead == "z_min"], 0.0,
""),
"z_max": cp.PyMeasurement("z_max", obj[cathead == "z_max"], 0.0,
"")
}
newSource.setParameters(newParamsDict)
initcatalog.insert(newSource)
moduleParametrizer = cp.PyModuleParametrisation()
moduleParametrizer.setFlags(Parameters["parameters"]["optimiseMask"],
Parameters["parameters"]["fitBusyFunction"])
cube = cube.astype("<f4", copy=False)
mask = mask.astype("<i2", copy=False)
moduleParametrizer.run(cube, mask, initcatalog)
results = moduleParametrizer.getCatalog()
# Append the results to the objects array or reset
replParam = [
"x_min", "x_max", "y_min", "y_max", "z_min", "z_max", "id", "x", "y",
"z", "n_pix"
]
origParam = [
"x_min", "x_max", "y_min", "y_max", "z_min", "z_max", "id", "x", "y",
"z", "n_pix"
]
d = results.getSources()
# Select data set with maximum number of parameters
parsListLen = [
len(d[list(d.keys())[i]].getParameters()) for i in range(0, len(d))
]
index = parsListLen.index(max(parsListLen))
# Add parameter names from parameterisation
pars = d[list(d.keys())[index]].getParameters()
cathead = list(cathead)
newunits = {
"id": "-",
"flag": "-",
"x": "pix",
"y": "pix",
"z": "pix",
"err_x": "pix",
"err_y": "pix",
"err_z": "pix",
"x_min": "pix",
"x_max": "pix",
"y_min": "pix",
"y_max": "pix",
"z_min": "chan",
"z_max": "chan",
"w50": "chan",
"w20": "chan",
"err_w50": "chan",
"err_w20": "chan",
"wm50": "chan",
"f_wm50": dunits,
"ell_maj": "pix",
"ell_min": "pix",
"ell_pa": "deg",
"ell3s_maj": "pix",
"ell3s_min": "pix",
"ell3s_pa": "deg",
"kin_pa": "deg",
"f_int": dunits,
"bf_flag": "-",
"bf_chi2": "-",
"bf_z": "chan",
"bf_a": dunits,
"bf_b1": "chan**(-1)",
"bf_b2": "chan**(-1)",
"bf_c": "chan**(-2)",
"bf_xe": "chan",
"bf_xp": "chan",
"bf_w": "chan",
"bf_w50": "chan",
"bf_w20": "chan",
"bf_f_peak": dunits,
"bf_f_int": dunits,
"rms": dunits,
"f_peak": dunits,
"snr_int": "-"
}
catformt = list(catformt)
catparunits = list(catparunits)
for i in sorted(pars):
if i not in replParam:
cathead.append(i)
catformt.append("%18.6e")
catparunits.append(newunits[i])
# Extend the parameter array
tmpObjects = np.empty((objects.shape[0], len(cathead)))
tmpObjects[:, :] = np.nan
tmpObjects[:, 0:objects.shape[1]] = objects
objects = tmpObjects
for i in d:
source_dict = d[i].getParameters()
# Check source index
ID = int(source_dict["id"].getValue())
IDind = cathead.index("id")
ind = np.where(objects[:, IDind] == ID)[0][0]
for j in sorted(source_dict):
if j in replParam:
objects[ind][cathead.index(origParam[replParam.index(
j)])] = source_dict[j].getValue()
else:
objects[ind][cathead.index(j)] = source_dict[j].getValue()
objects = np.array(objects)
cathead = np.array(cathead)
catparunits = np.array(catparunits)
catformt = np.array(catformt)
# if mask optimization is enabled, some parameters from the linker have to be updated
if Parameters["parameters"]["optimiseMask"]:
for i in range(objects.shape[0]):
# bounding box coordinates
coord = []
for c in ["x_min", "x_max", "y_min", "y_max", "z_min", "z_max"]:
coord.append(int(objects[i, cathead == c]))
# cut out object submask
submask = mask[coord[4]:coord[5] + 1, coord[2]:coord[3] + 1,
coord[0]:coord[1] + 1]
objID = objects[i, cathead == "id"]
submask[submask != objID] = 0
# Update n_pix, x_geo and n_chan
n_pix = submask.sum() / objID
ind = np.vstack(np.where(submask == objID))
cgeo = (ind.sum(axis=1)).astype(float) / float(n_pix)
x_geo, y_geo, z_geo = cgeo[2] + coord[0], cgeo[1] + coord[2], cgeo[
0] + coord[4]
zmin, zmax = min(ind[0]), max(ind[0]) + 1
n_chan = zmax - zmin
# Update n_los
submaskSumA0 = submask.sum(axis=0)
submaskSumA0[submaskSumA0 > 1] = 1
n_los = submaskSumA0.sum()
objects[i, cathead == "n_pix"] = n_pix
objects[i, cathead == "n_chan"] = n_chan
objects[i, cathead == "n_los"] = n_los
objects[i, cathead == "x_geo"] = x_geo
objects[i, cathead == "y_geo"] = y_geo
objects[i, cathead == "z_geo"] = z_geo
del submask
err.message("Parameterisation complete.")
return cube, mask, objects, cathead, catformt, catparunits
def testConstructor(self): print 'Testing PySourceCatalog: constructor' sc1 = cp.PySourceCatalog() sc2 = cp.PySourceCatalog(sc1)