def recompute_mag(mpc_obs):
"""
Get the mag of the object given the mpc.Observation
"""
# TODO this really shouldn't need to build a 'reading' to get the cutout...
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if parts is None:
return None
expnum = parts.group('expnum')
ccd = parts.group('ccd')
ftype = parts.group('type')
observation = astrom.Observation(expnum,ftype,ccd)
observation.header = storage.get_mopheader(int(expnum),int(ccd))
reading = astrom.SourceReading(float(mpc_obs.comment.X),
float(mpc_obs.comment.Y),
float(mpc_obs.comment.X),
float(mpc_obs.comment.Y),
mpc_obs.coordinate.ra.degrees,
mpc_obs.coordinate.dec.degrees,
float(mpc_obs.comment.X),
float(mpc_obs.comment.Y),
observation,
ssos=True,
from_input_file=True,null_observation=False,discovery=mpc_obs.discovery)
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
try:
(x, y, mag, merr) = cutout.get_observed_magnitude()
mpc_obs.mag = mag
mpc_obs.mag_err = merr
except:
mpc_obs.mag = None
mpc_obs.mag_err = None
return
"""
An example of parsing an .astrom file, downloading one of the images,
doing photometry calculations on it, and then displaying the image.
"""
from ossos import astrom
from ossos import fitsviewer
from ossos.downloads.cutouts import ImageCutoutDownloader
sources = astrom.parse_sources(
"../tests/data/astromdir/realstest2.measure3.reals.astrom")
reading = sources[0].get_reading(0)
downloader = ImageCutoutDownloader(slice_rows=50, slice_cols=50)
cutout = downloader.download_cutout(reading, needs_apcor=True)
print("RA: %f" % cutout.ra)
print("DEC: %f" % cutout.dec)
print("Observed magnitude: %f" % cutout.get_observed_magnitude()[0])
fitsviewer.display_hdulist(cutout.hdulist)
class CutoutDownloaderTest(FileReadingTestCase):
def setUp(self):
self.image_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/1584431p15.fits"
self.apcor_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/ccd15/1584431p15.apcor"
self.reading = MagicMock(spec=SourceReading)
self.reading.x = 0
self.reading.y = 0
self.reading.ra = 0
self.reading.dec = 0
self.reading.get_image_uri.return_value = self.image_uri
self.reading.get_apcor_uri.return_value = self.apcor_uri
self.reading.get_extension.return_value = 19
self.reading.get_original_image_size.return_value = (2000, 3000)
self.reading.is_inverted.return_value = False
self.needs_apcor = True
self.focus = (75, 80)
self.vosclient = Mock(spec=vos.Client)
self.downloader = ImageCutoutDownloader(slice_rows=100,
slice_cols=50,
vosclient=self.vosclient)
# Mock vosclient to open a local file instead of one from vospace
self.localfile = open(self.get_abs_path("data/testimg.fits"), "rb")
self.apcorfile = tempfile.TemporaryFile("r+b")
self.apcorfile.write("4 10 0.3 0.1")
self.apcorfile.flush()
self.apcorfile.seek(0)
def choose_ret_val(*args, **kwargs):
selected_file = None
if self.image_uri in args:
selected_file = self.localfile
elif self.apcor_uri in args:
selected_file = self.apcorfile
else:
self.fail("Unrecognized URI")
selected_file.seek(0)
return selected_file
self.vosclient.open.side_effect = choose_ret_val
def tearDown(self):
self.localfile.close()
self.apcorfile.close()
def test_request_image_cutout(self):
download = self.downloader.download_cutout(
self.reading, focus=self.focus, needs_apcor=self.needs_apcor)
assert_that(
self.vosclient.open.call_args_list,
contains(
call(self.image_uri,
view="cutout",
cutout="[19][50:100,30:130]"),
call(self.apcor_uri, view="data")))
# This is just a test file, make sure we can read an expected value
# it. It won't have the right shape necessarily though.
assert_that(download.fits_header["FILENAME"],
equal_to("u5780205r_cvt.c0h"))
def match_planted(astrom_filename, match_filename, false_positive_filename):
"""
Using the astrom_filename as input get the Object.planted file from VOSpace and match
planted sources with found sources.
The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the
first exposure as read from the .astrom file.
:param astrom_filename: name of the fk*reals.astrom file to check against Object.planted
:param match_filename: a file that will contain a list of all planted sources and the matched found source
:param false_positive_filename: .astrom format output containing input objects that had no match in planted
"""
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
fk_candidate_observations = astrom.parse(astrom_filename)
matches_fptr = storage.open_vos_or_local(match_filename,'w')
objects_planted_uri = fk_candidate_observations.observations[0].get_object_planted_uri()
objects_planted = image_slice_downloader.download_raw(objects_planted_uri, view='data').split('\n')
planted_objects = []
for line in objects_planted[1:]:
if len(line) == 0 or line[0] == '#':
continue
planted_objects.append(PlantedObject(line))
false_positives_stream_writer = None
matches_fptr.write("#{}\n".format(fk_candidate_observations.observations[0].rawname))
matches_fptr.write("{:1s}{} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}\n".format(
"",objects_planted[0],"x_dao","y_dao","mag_dao","merr_dao", "rate_mes", "ang_mes", "dr_pixels" ))
found_idxs = []
for source in fk_candidate_observations.get_sources():
reading = source.get_reading(0)
third = source.get_reading(2)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
try:
(x, y, mag, merr) = cutout.get_observed_magnitude()
except TaskError as e:
logger.warning(str(e))
mag = 0.0
merr = -1.0
matched = None
for idx in range(len(planted_objects)):
planted_object = planted_objects[idx]
dist = math.sqrt((reading.x-planted_object.x)**2 + (reading.y - planted_object.y)**2)
if matched is None or dist < matched:
matched = dist
matched_object_idx = idx
start_jd = Time(reading.obs.header['MJD_OBS_CENTER'],format='mpc', scale='utc').jd
end_jd = Time(third.obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd
exptime = float(reading.obs.header['EXPTIME'])
rate = math.sqrt((third.x - reading.x)**2 + (third.y - reading.y)**2)/(
24*(end_jd - start_jd) )
angle = math.degrees(math.atan2(third.y - reading.y,third.x - reading.x))
if matched > 3*rate*exptime/3600.0 and False :
# this is a false positive (candidate not near artificial source)
# create a .astrom style line for feeding to validate for checking later
if false_positives_ftpr is None or false_positives_stream_writer is None:
# create false positive file for storing results
false_positives_ftpr = open(false_positive_filename,'w+')
false_positives_stream_writer = StreamingAstromWriter(
false_positives_ftpr,fk_candidate_observations.sys_header)
false_positives_stream_writer.write_source(source)
false_positives_ftpr.flush()
continue
elif matched_object_idx in found_idxs:
repeat = '#'
else:
repeat = ' '
found_idxs.append(matched_object_idx)
mags = []
merrs = []
for this_reading in source.get_readings()[1:]:
cutout = image_slice_downloader.download_cutout(this_reading, needs_apcor=True)
try:
(this_x, this_y, this_mag, this_merr) = cutout.get_observed_magnitude()
except TaskError as e:
logger.warning(str(e))
this_mag = 0.0
this_merr = -1.0
mags.append(this_mag)
merrs.append(this_merr)
matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format(
repeat,
str(planted_objects[matched_object_idx]), reading.x, reading.y, mag, merr, rate, angle, matched))
for idx in range(len(mags)):
matches_fptr.write("{:8.2f} {:8.2f}".format(mags[idx], merrs[idx]))
matches_fptr.write("\n")
# close the false_positives
if false_positives_ftpr is not None:
false_positives_ftpr.close()
# record the unmatched Object.planted entries, for use in efficiency computing
for idx in range(len(planted_objects)):
if idx not in found_idxs:
planted_object = planted_objects[idx]
matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f}\n".format("",str(planted_object),
0, 0, 0, 0, 0, 0, 0))
matches_fptr.close()
"""
An example of parsing an .astrom file, downloading one of the images,
doing photometry calculations on it, and then displaying the image.
"""
from ossos import astrom
from ossos import fitsviewer
from ossos.downloads.cutouts import ImageCutoutDownloader
sources = astrom.parse_sources("../tests/data/astromdir/realstest2.measure3.reals.astrom")
reading = sources[0].get_reading(0)
downloader = ImageCutoutDownloader(slice_rows=50, slice_cols=50)
cutout = downloader.download_cutout(reading, needs_apcor=True)
print "RA: %f" % cutout.ra
print "DEC: %f" % cutout.dec
print "Observed magnitude: %f" % cutout.get_observed_magnitude()[0]
fitsviewer.display_hdulist(cutout.hdulist)
class CutoutDownloaderTest(FileReadingTestCase):
def setUp(self):
self.image_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/1584431p15.fits"
self.apcor_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/ccd15/1584431p15.apcor"
self.reading = MagicMock(spec=SourceReading)
self.reading.x = 0
self.reading.y = 0
self.reading.ra = 0
self.reading.dec = 0
self.reading.get_image_uri.return_value = self.image_uri
self.reading.get_apcor_uri.return_value = self.apcor_uri
self.reading.get_extension.return_value = 19
self.reading.get_original_image_size.return_value = (2000, 3000)
self.reading.is_inverted.return_value = False
self.needs_apcor = True
self.focus = (75, 80)
self.vosclient = Mock(spec=vos.Client)
self.downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=50,
vosclient=self.vosclient)
# Mock vosclient to open a local file instead of one from vospace
self.localfile = open(self.get_abs_path("data/testimg.fits"), "rb")
self.apcorfile = tempfile.TemporaryFile("r+b")
self.apcorfile.write("4 10 0.3 0.1")
self.apcorfile.flush()
self.apcorfile.seek(0)
def choose_ret_val(*args, **kwargs):
selected_file = None
if self.image_uri in args:
selected_file = self.localfile
elif self.apcor_uri in args:
selected_file = self.apcorfile
else:
self.fail("Unrecognized URI")
selected_file.seek(0)
return selected_file
self.vosclient.open.side_effect = choose_ret_val
def tearDown(self):
self.localfile.close()
self.apcorfile.close()
def test_request_image_cutout(self):
download = self.downloader.download_cutout(self.reading,
focus=self.focus,
needs_apcor=self.needs_apcor)
assert_that(self.vosclient.open.call_args_list, contains(
call(self.image_uri, view="cutout", cutout="[19][50:100,30:130]"),
call(self.apcor_uri, view="data")
))
# This is just a test file, make sure we can read an expected value
# it. It won't have the right shape necessarily though.
assert_that(download.fits_header["FILENAME"],
equal_to("u5780205r_cvt.c0h"))
def recompute_mag(mpc_in):
"""
Get the mag of the object given the mpc.Observation
"""
# TODO this really shouldn't need to build a 'reading' to get the cutout...
from ossos.downloads.cutouts import ImageCutoutDownloader
mpc_obs = deepcopy(mpc_in)
assert isinstance(mpc_obs, mpc.Observation)
if mpc_obs.null_observation:
return mpc_obs
parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame)
if parts is None:
return mpc_obs
expnum = parts.group('expnum')
ccd = parts.group('ccd')
file_type = parts.group('type')
observation = astrom.Observation(expnum, file_type, ccd)
assert isinstance(observation, astrom.Observation)
ast_header = storage.get_astheader(int(expnum), int(ccd))
# The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file.
new_zp = ast_header.get('PHOTZP')
# The ZP in the image header is likely the one used for the original photometry.
old_zp = storage.get_zeropoint(int(expnum), int(ccd))
reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x),
float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degrees,
mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.x),
float(mpc_obs.comment.y),
observation, ssos=True, from_input_file=True, null_observation=False,
discovery=mpc_obs.discovery)
# reading.is_inverted = reading.compute_inverted()
reading._header = storage.get_mopheader(expnum, ccd)
reading._header['MAXCOUNT'] = 30000.0
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
cutout.zmag = new_zp
if math.fabs(new_zp - old_zp) > 0.3:
logging.warning("Large change in zeropoint detected: {} -> {}".format(old_zp, new_zp))
try:
(x, y, mag, merr) = cutout.get_observed_magnitude(zmag=new_zp)
(x, y) = cutout.get_observed_coordinates((x, y))
except:
logging.warn("Failed to do photometry.")
return mpc_obs
if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty:
logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag))
if math.sqrt((x - mpc_obs.comment.x) ** 2 + (y - mpc_obs.comment.y) ** 2) > 1.0:
logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x,
mpc_obs.comment.y,
x,
y))
# Don't use the new X/Y for Hand measured entries.
if str(mpc_obs.note1) != "H":
mpc_obs.comment.x = x
mpc_obs.comment.y = y
mpc_obs.comment.mag = mag
mpc_obs.comment.mag_uncertainty = merr
if mpc_obs.mag is not None:
mpc_obs.mag = mag
return mpc_obs