def insert_extracted_sources(image_id, results, extract_type,
ff_runcat_ids=None, ff_monitor_ids=None):
"""
Insert all detections from sourcefinder into the extractedsource table.
Besides the source properties from sourcefinder, we calculate additional
attributes that are increase performance in other tasks.
The strict sequence from results (the sourcefinder detections) is given below.
Note the units between sourcefinder and database.
(0) ra [deg], (1) dec [deg],
(2) ra_fit_err [deg], (3) decl_fit_err [deg],
(4) peak_flux [Jy], (5) peak_flux_err [Jy],
(6) int_flux [Jy], (7) int_flux_err [Jy],
(8) significance detection level,
(9) beam major width (arcsec), (10) - minor width (arcsec), (11) - parallactic angle [deg],
(12) ew_sys_err [arcsec], (13) ns_sys_err [arcsec],
(14) error_radius [arcsec]
(15) gaussian fit (bool)
(16), (17) chisq, reduced_chisq (float)
ra_fit_err and decl_fit_err are the 1-sigma errors from the gaussian fit,
in degrees. Note that for a source located towards the poles the ra_fit_err
increases with absolute declination.
error_radius is a pessimistic on-sky error estimate in arcsec.
ew_sys_err and ns_sys_err represent the telescope dependent systematic errors
and are in arcsec.
An on-sky error (declination independent, and used in de ruiter calculations)
is then:
uncertainty_ew^2 = ew_sys_err^2 + error_radius^2
uncertainty_ns^2 = ns_sys_err^2 + error_radius^2
The units of uncertainty_ew and uncertainty_ns are in degrees.
The error on RA is given by ra_err. For a source with an RA of ra and an error
of ra_err, its RA lies in the range [ra-ra_err, ra+ra_err].
ra_err^2 = ra_fit_err^2 + [alpha_inflate(ew_sys_err,decl)]^2
decl_err^2 = decl_fit_err^2 + ns_sys_err^2.
The units of ra_err and decl_err are in degrees.
Here alpha_inflate() is the RA inflation function, it converts an
angular on-sky distance to a ra distance at given declination.
Input argument "extract" tells whether the source detections originate from:
'blind': blind source extraction
'ff_nd': from forced fits at null detection locations
'ff_ms': from forced fits at monitoringlist positions
Input argument ff_runcat is not empty in the case of forced fits from
null detections. It contains the runningcatalog ids from which the
source positions were derived for the forced fits. In that case the
runcat ids will be inserted into the extractedsource table as well,
to simplify further null-detection processing.
For blind extractions this list is empty (None).
For all extracted sources additional parameters are calculated,
and appended to the sourcefinder data. Appended and converted are:
- the image id to which the extracted sources belong to
- the zone in which an extracted source falls is calculated, based
on its declination. We adopt a zoneheight of 1 degree, so
the floor of the declination represents the zone.
- the positional errors are converted from degrees to arcsecs
- the Cartesian coordinates of the source position
- ra * cos(radians(decl)), this is very often being used in
source-distance calculations
"""
if not len(results):
logger.info("No extract_type=%s sources added to extractedsource for"
" image %s" % (extract_type, image_id))
return
xtrsrc = []
for i, src in enumerate(results):
r = list(src)
# Drop any fits with infinite flux errors
if math.isinf(r[5]) or math.isinf(r[7]):
logger.warn("Dropped source fit with infinite flux errors "
"at position %s %s" % (r[0], r[1]))
continue
# Use 360 degree rather than infinite uncertainty for
# unconstrained positions.
r[14] = substitute_inf(r[14], 360.0)
r[15] = int(r[15])
# ra_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[2]**2 + alpha_inflate(r[12]/3600., r[1])**2))
# decl_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[3]**2 + (r[13]/3600.)**2))
# uncertainty_ew: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[12]**2 + r[14]**2)/3600.)
# uncertainty_ns: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[13]**2 + r[14]**2)/3600.)
r.append(image_id) # id of the image
r.append(int(math.floor(r[1]))) # zone
r.extend(eq_to_cart(r[0], r[1])) # Cartesian x,y,z
r.append(r[0] * math.cos(math.radians(r[1]))) # ra * cos(radians(decl))
if extract_type == 'blind':
r.append(0)
elif extract_type == 'ff_nd':
r.append(1)
elif extract_type == 'ff_ms':
r.append(2)
else:
raise ValueError("Not a valid extractedsource insert type: '%s'"
% extract_type)
if ff_runcat_ids is not None:
assert len(results)==len(ff_runcat_ids)
r.append(ff_runcat_ids[i])
else:
r.append(None)
if ff_monitor_ids is not None:
assert len(results)==len(ff_monitor_ids)
r.append(ff_monitor_ids[i])
else:
r.append(None)
xtrsrc.append(r)
insertion_query = """\
INSERT INTO extractedsource
(ra
,decl
,ra_fit_err
,decl_fit_err
,f_peak
,f_peak_err
,f_int
,f_int_err
,det_sigma
,semimajor
,semiminor
,pa
,ew_sys_err
,ns_sys_err
,error_radius
,fit_type
,chisq
,reduced_chisq
,ra_err
,decl_err
,uncertainty_ew
,uncertainty_ns
,image
,zone
,x
,y
,z
,racosdecl
,extract_type
,ff_runcat
,ff_monitor
)
VALUES {placeholder}
"""
if xtrsrc:
cols_per_row = len(xtrsrc[0])
placeholder_per_row = '('+ ','.join(['%s']*cols_per_row) +')'
placeholder_full = ','.join([placeholder_per_row]*len(xtrsrc))
query = insertion_query.format(placeholder= placeholder_full)
cursor = tkp.db.execute(query, tuple(itertools.chain.from_iterable(xtrsrc)),
commit=True)
insert_num = cursor.rowcount
#if insert_num == 0:
# logger.info("No forced-fit sources added to extractedsource for "
# "image %s" % (image_id,))
if extract_type == 'blind':
logger.info("Inserted %d sources in extractedsource for image %s" %
(insert_num, image_id))
elif extract_type == 'ff_nd':
logger.info("Inserted %d forced-fit null detections in extractedsource"
" for image %s" % (insert_num, image_id))
elif extract_type == 'ff_ms':
logger.info("Inserted %d forced-fit for monitoring in extractedsource"
" for image %s" % (insert_num, image_id))
def insert_extracted_sources(image_id, results, extract_type='blind',
ff_runcat_ids=None, ff_monitor_ids=None):
"""
Insert all detections from sourcefinder into the extractedsource table.
Besides the source properties from sourcefinder, we calculate additional
attributes that are increase performance in other tasks.
The strict sequence from results (the sourcefinder detections) is given below.
Note the units between sourcefinder and database.
(0) ra [deg], (1) dec [deg],
(2) ra_fit_err [deg], (3) decl_fit_err [deg],
(4) peak_flux [Jy], (5) peak_flux_err [Jy],
(6) int_flux [Jy], (7) int_flux_err [Jy],
(8) significance detection level,
(9) beam major width (arcsec), (10) - minor width (arcsec), (11) - parallactic angle [deg],
(12) ew_sys_err [arcsec], (13) ns_sys_err [arcsec],
(14) error_radius [arcsec]
(15) gaussian fit (bool)
(16), (17) chisq, reduced_chisq (float)
ra_fit_err and decl_fit_err are the 1-sigma errors from the gaussian fit,
in degrees. Note that for a source located towards the poles the ra_fit_err
increases with absolute declination.
error_radius is a pessimistic on-sky error estimate in arcsec.
ew_sys_err and ns_sys_err represent the telescope dependent systematic errors
and are in arcsec.
An on-sky error (declination independent, and used in de ruiter calculations)
is then:
uncertainty_ew^2 = ew_sys_err^2 + error_radius^2
uncertainty_ns^2 = ns_sys_err^2 + error_radius^2
The units of uncertainty_ew and uncertainty_ns are in degrees.
The error on RA is given by ra_err. For a source with an RA of ra and an error
of ra_err, its RA lies in the range [ra-ra_err, ra+ra_err].
ra_err^2 = ra_fit_err^2 + [alpha_inflate(ew_sys_err,decl)]^2
decl_err^2 = decl_fit_err^2 + ns_sys_err^2.
The units of ra_err and decl_err are in degrees.
Here alpha_inflate() is the RA inflation function, it converts an
angular on-sky distance to a ra distance at given declination.
Input argument "extract" tells whether the source detections originate from:
'blind': blind source extraction
'ff_nd': from forced fits at null detection locations
'ff_ms': from forced fits at monitoringlist positions
Input argument ff_runcat is not empty in the case of forced fits from
null detections. It contains the runningcatalog ids from which the
source positions were derived for the forced fits. In that case the
runcat ids will be inserted into the extractedsource table as well,
to simplify further null-detection processing.
For blind extractions this list is empty (None).
For all extracted sources additional parameters are calculated,
and appended to the sourcefinder data. Appended and converted are:
- the image id to which the extracted sources belong to
- the zone in which an extracted source falls is calculated, based
on its declination. We adopt a zoneheight of 1 degree, so
the floor of the declination represents the zone.
- the positional errors are converted from degrees to arcsecs
- the Cartesian coordinates of the source position
- ra * cos(radians(decl)), this is very often being used in
source-distance calculations
"""
if not len(results):
logger.debug("No extract_type=%s sources added to extractedsource for"
" image %s" % (extract_type, image_id))
return
xtrsrc = []
for i, src in enumerate(results):
r = list(src)
# Drop any fits with infinite flux errors
if math.isinf(r[5]) or math.isinf(r[7]):
logger.warn("Dropped source fit with infinite flux errors "
"at position {} {} in image {}".format(
r[0], r[1], image_id))
continue
# Use 360 degree rather than infinite uncertainty for
# unconstrained positions.
r[14] = substitute_inf(r[14], 360.0)
r[15] = int(r[15])
# ra_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[2]**2 + alpha_inflate(r[12]/3600., r[1])**2))
# decl_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[3]**2 + (r[13]/3600.)**2))
# uncertainty_ew: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[12]**2 + r[14]**2)/3600.)
# uncertainty_ns: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[13]**2 + r[14]**2)/3600.)
r.append(image_id) # id of the image
r.append(int(math.floor(r[1]))) # zone
r.extend(eq_to_cart(r[0], r[1])) # Cartesian x,y,z
r.append(r[0] * math.cos(math.radians(r[1]))) # ra * cos(radians(decl))
if extract_type == 'blind':
r.append(0)
elif extract_type == 'ff_nd':
r.append(1)
elif extract_type == 'ff_ms':
r.append(2)
else:
raise ValueError("Not a valid extractedsource insert type: '%s'"
% extract_type)
if ff_runcat_ids is not None:
assert len(results)==len(ff_runcat_ids)
r.append(ff_runcat_ids[i])
else:
r.append(None)
if ff_monitor_ids is not None:
assert len(results)==len(ff_monitor_ids)
r.append(ff_monitor_ids[i])
else:
r.append(None)
xtrsrc.append(r)
insertion_query = """\
INSERT INTO extractedsource
(ra
,decl
,ra_fit_err
,decl_fit_err
,f_peak
,f_peak_err
,f_int
,f_int_err
,det_sigma
,semimajor
,semiminor
,pa
,ew_sys_err
,ns_sys_err
,error_radius
,fit_type
,chisq
,reduced_chisq
,ra_err
,decl_err
,uncertainty_ew
,uncertainty_ns
,image
,zone
,x
,y
,z
,racosdecl
,extract_type
,ff_runcat
,ff_monitor
)
VALUES {placeholder}
"""
if xtrsrc:
cols_per_row = len(xtrsrc[0])
placeholder_per_row = '('+ ','.join(['%s']*cols_per_row) +')'
placeholder_full = ','.join([placeholder_per_row]*len(xtrsrc))
query = insertion_query.format(placeholder= placeholder_full)
cursor = tkp.db.execute(query, tuple(itertools.chain.from_iterable(xtrsrc)),
commit=True)
insert_num = cursor.rowcount
#if insert_num == 0:
# logger.info("No forced-fit sources added to extractedsource for "
# "image %s" % (image_id,))
if extract_type == 'blind':
logger.debug("Inserted %d sources in extractedsource for image %s" %
(insert_num, image_id))
elif extract_type == 'ff_nd':
logger.debug("Inserted %d forced-fit null detections in extractedsource"
" for image %s" % (insert_num, image_id))
elif extract_type == 'ff_ms':
logger.debug("Inserted %d forced-fit for monitoring in extractedsource"
" for image %s" % (insert_num, image_id))
def insert_extracted_sources(image_id, results, extract):
"""Insert all detections from sourcefinder into the extractedsource table.
Besides the source properties from sourcefinder, we calculate additional
attributes that are increase performance in other tasks.
The strict sequence from results (the sourcefinder detections) is given below.
Note the units between sourcefinder and database.
(0) ra [deg], (1) dec [deg],
(2) ra_fit_err [deg], (3) decl_fit_err [deg],
(4) peak_flux [Jy], (5) peak_flux_err [Jy],
(6) int_flux [Jy], (7) int_flux_err [Jy],
(8) significance detection level,
(9) beam major width (arcsec), (10) - minor width (arcsec), (11) - parallactic angle [deg],
(12) ew_sys_err [arcsec], (13) ns_sys_err [arcsec],
(14) error_radius [arcsec]
ra_fit_err and decl_fit_err are the 1-sigma errors from the gaussian fit,
in degrees. Note that for a source located towards the poles the ra_fit_err
increases with absolute declination.
error_radius is a pessimistic on-sky error estimate in arcsec.
ew_sys_err and ns_sys_err represent the telescope dependent systematic errors
and are in arcsec.
An on-sky error (declination independent, and used in de ruiter calculations)
is then:
uncertainty_ew^2 = ew_sys_err^2 + error_radius^2
uncertainty_ns^2 = ns_sys_err^2 + error_radius^2
The units of uncertainty_ew and uncertainty_ns are in degrees.
The error on RA is given by ra_err. For a source with an RA of ra and an error
of ra_err, its RA lies in the range [ra-ra_err, ra+ra_err].
ra_err^2 = ra_fit_err^2 + [alpha_inflate(ew_sys_err,decl)]^2
decl_err^2 = decl_fit_err^2 + ns_sys_err^2.
The units of ra_err and decl_err are in degrees.
Here alpha_inflate() is the RA inflation function, it converts an
angular on-sky distance to a ra distance at given declination.
Input argument "extract" tells whether the source detections originate from:
0: blind source extraction
1: from forced fits at null detection locations
2: from forced fits at monitoringlist positions
For all extracted sources additional parameters are calculated,
and appended to the sourcefinder data. Appended and converted are:
- the image id to which the extracted sources belong to
- the zone in which an extracted source falls is calculated, based
on its declination. We adopt a zoneheight of 1 degree, so
the floor of the declination represents the zone.
- the positional errors are converted from degrees to arcsecs
- the Cartesian coordinates of the source position
- ra * cos(radians(decl)), this is very often being used in
source-distance calculations
"""
if not len(results):
logger.info("No extract_type=%s sources added to extractedsource for"
" image %s" % (extract, image_id))
return
xtrsrc = []
for src in results:
r = list(src)
# Use 360 degree rather than infinite uncertainty for
# unconstrained positions.
r[14] = substitute_inf(r[14], 360.0)
# ra_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[2]**2 + alpha_inflate(r[12]/3600., r[1])**2))
# decl_err: sqrt of quadratic sum of fitted and systematic errors.
r.append(math.sqrt(r[3]**2 + (r[13]/3600.)**2))
# uncertainty_ew: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[12]**2 + r[14]**2)/3600.)
# uncertainty_ns: sqrt of quadratic sum of systematic error and error_radius
# divided by 3600 because uncertainty in degrees and others in arcsec.
r.append(math.sqrt(r[13]**2 + r[14]**2)/3600.)
r.append(image_id) # id of the image
r.append(int(math.floor(r[1]))) # zone
r.extend(eq_to_cart(r[0], r[1])) # Cartesian x,y,z
r.append(r[0] * math.cos(math.radians(r[1]))) # ra * cos(radians(decl))
if extract == 'blind':
r.append(0)
elif extract == 'ff_nd':
r.append(1)
elif extract == 'ff_ms':
r.append(2)
else:
raise ValueError("Not a valid extractedsource insert type: '%s'" % extract)
xtrsrc.append(r)
values = [str(tuple(xsrc)) for xsrc in xtrsrc]
query = """\
INSERT INTO extractedsource
(ra
,decl
,ra_fit_err
,decl_fit_err
,f_peak
,f_peak_err
,f_int
,f_int_err
,det_sigma
,semimajor
,semiminor
,pa
,ew_sys_err
,ns_sys_err
,error_radius
,ra_err
,decl_err
,uncertainty_ew
,uncertainty_ns
,image
,zone
,x
,y
,z
,racosdecl
,extract_type
)
VALUES
""" + ",".join(values)
cursor = tkp.db.execute(query, commit=True)
insert_num = cursor.rowcount
if insert_num == 0:
logger.info("No forced-fit sources added to extractedsource for "
"image %s" % (image_id,))
elif extract == 'blind':
logger.info("Inserted %d sources in extractedsource for image %s" %
(insert_num, image_id))
elif extract == 'ff_nd':
logger.info("Inserted %d forced-fit null detections in extractedsource"
" for image %s" % (insert_num, image_id))
elif extract == 'ff_ms':
logger.info("Inserted %d forced-fit for monitoring sourcs in extractedsource"
" for image %s" % (insert_num, image_id))