def setUp(self): self.image_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/1584431p15.fits" self.apcor_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/ccd15/1584431p15.apcor" self.reading = MagicMock(spec=SourceReading) self.reading.x = 0 self.reading.y = 0 self.reading.ra = 0 self.reading.dec = 0 self.reading.get_image_uri.return_value = self.image_uri self.reading.get_apcor_uri.return_value = self.apcor_uri self.reading.get_extension.return_value = 19 self.reading.get_original_image_size.return_value = (2000, 3000) self.reading.is_inverted.return_value = False self.needs_apcor = True self.focus = (75, 80) self.vosclient = Mock(spec=vos.Client) self.downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=50, vosclient=self.vosclient) # Mock vosclient to open a local file instead of one from vospace self.localfile = open(self.get_abs_path("data/testimg.fits"), "rb") self.apcorfile = tempfile.TemporaryFile("r+b") self.apcorfile.write("4 10 0.3 0.1") self.apcorfile.flush() self.apcorfile.seek(0) def choose_ret_val(*args, **kwargs): selected_file = None if self.image_uri in args: selected_file = self.localfile elif self.apcor_uri in args: selected_file = self.apcorfile else: self.fail("Unrecognized URI") selected_file.seek(0) return selected_file self.vosclient.open.side_effect = choose_ret_val
def get_planted_objects(mpc_comment): """ Using an mpc comment record determine the location of the Object.planted file and retrieve. then build the file lines into an array of plantedObject objects. """ assert isinstance(mpc_comment, mpc.MPCComment) (expnum, ccd) = mpc_comment.frame.splt('p') obs = astrom.Observation.from_source_reference(int(expnum), int(ccd), float(mpc_comment.X), float(mpc_comment.Y)) object_planted_uri = obs.get_object_planted_uri() image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) planted_objects = [] for line in image_slice_downloader.download_raw(object_planted_uri, view='data').splt('\n')[1:]: planted_objects.append(PlantedObject(line)) return planted_objects
def recompute_mag(mpc_obs): """ Get the mag of the object given the mpc.Observation """ # TODO this really shouldn't need to build a 'reading' to get the cutout... parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if parts is None: return None expnum = parts.group('expnum') ccd = parts.group('ccd') ftype = parts.group('type') observation = astrom.Observation(expnum,ftype,ccd) observation.header = storage.get_mopheader(int(expnum),int(ccd)) reading = astrom.SourceReading(float(mpc_obs.comment.X), float(mpc_obs.comment.Y), float(mpc_obs.comment.X), float(mpc_obs.comment.Y), mpc_obs.coordinate.ra.degrees, mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.X), float(mpc_obs.comment.Y), observation, ssos=True, from_input_file=True,null_observation=False,discovery=mpc_obs.discovery) image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) try: (x, y, mag, merr) = cutout.get_observed_magnitude() mpc_obs.mag = mag mpc_obs.mag_err = merr except: mpc_obs.mag = None mpc_obs.mag_err = None return
""" An example of parsing an .astrom file, downloading one of the images, doing photometry calculations on it, and then displaying the image. """ from ossos import astrom from ossos import fitsviewer from ossos.downloads.cutouts import ImageCutoutDownloader sources = astrom.parse_sources( "../tests/data/astromdir/realstest2.measure3.reals.astrom") reading = sources[0].get_reading(0) downloader = ImageCutoutDownloader(slice_rows=50, slice_cols=50) cutout = downloader.download_cutout(reading, needs_apcor=True) print("RA: %f" % cutout.ra) print("DEC: %f" % cutout.dec) print("Observed magnitude: %f" % cutout.get_observed_magnitude()[0]) fitsviewer.display_hdulist(cutout.hdulist)
from astropy.io import ascii from astropy.table import MaskedColumn, Table logger = logging.getLogger('vos') logger.setLevel(logging.CRITICAL) logger.addHandler(logging.StreamHandler()) BRIGHT_LIMIT = 23.0 OBJECT_PLANTED = "Object.planted" MINIMUM_BRIGHT_DETECTIONS = 5 MINIMUM_BRIGHT_FRACTION = 0.5 # This image_slice_downloader handles pulling a small cutout of the image from VOSpace so we can do photometry # on this small chunk of the image. image_slice_downloader = ImageCutoutDownloader(slice_rows=60, slice_cols=60) def measure_mags(measures, table_row): try: start_jd = Time(measures[0].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd end_jd = Time(measures[-1].obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd table_row['measure_x'] = measures[0].x table_row['measure_y'] = measures[0].y rate = math.sqrt((measures[-1].x - measures[0].x) ** 2 + (measures[-1].y - measures[0].y) ** 2) / ( 24 * (end_jd - start_jd)) rate = int(rate * 100) / 100.0 table_row['measure_rate'] = rate
class CutoutDownloaderTest(FileReadingTestCase): def setUp(self): self.image_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/1584431p15.fits" self.apcor_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/ccd15/1584431p15.apcor" self.reading = MagicMock(spec=SourceReading) self.reading.x = 0 self.reading.y = 0 self.reading.ra = 0 self.reading.dec = 0 self.reading.get_image_uri.return_value = self.image_uri self.reading.get_apcor_uri.return_value = self.apcor_uri self.reading.get_extension.return_value = 19 self.reading.get_original_image_size.return_value = (2000, 3000) self.reading.is_inverted.return_value = False self.needs_apcor = True self.focus = (75, 80) self.vosclient = Mock(spec=vos.Client) self.downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=50, vosclient=self.vosclient) # Mock vosclient to open a local file instead of one from vospace self.localfile = open(self.get_abs_path("data/testimg.fits"), "rb") self.apcorfile = tempfile.TemporaryFile("r+b") self.apcorfile.write("4 10 0.3 0.1") self.apcorfile.flush() self.apcorfile.seek(0) def choose_ret_val(*args, **kwargs): selected_file = None if self.image_uri in args: selected_file = self.localfile elif self.apcor_uri in args: selected_file = self.apcorfile else: self.fail("Unrecognized URI") selected_file.seek(0) return selected_file self.vosclient.open.side_effect = choose_ret_val def tearDown(self): self.localfile.close() self.apcorfile.close() def test_request_image_cutout(self): download = self.downloader.download_cutout( self.reading, focus=self.focus, needs_apcor=self.needs_apcor) assert_that( self.vosclient.open.call_args_list, contains( call(self.image_uri, view="cutout", cutout="[19][50:100,30:130]"), call(self.apcor_uri, view="data"))) # This is just a test file, make sure we can read an expected value # it. It won't have the right shape necessarily though. assert_that(download.fits_header["FILENAME"], equal_to("u5780205r_cvt.c0h"))
def match_planted(astrom_filename, match_filename, false_positive_filename): """ Using the astrom_filename as input get the Object.planted file from VOSpace and match planted sources with found sources. The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the first exposure as read from the .astrom file. :param astrom_filename: name of the fk*reals.astrom file to check against Object.planted :param match_filename: a file that will contain a list of all planted sources and the matched found source :param false_positive_filename: .astrom format output containing input objects that had no match in planted """ image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) fk_candidate_observations = astrom.parse(astrom_filename) matches_fptr = storage.open_vos_or_local(match_filename,'w') objects_planted_uri = fk_candidate_observations.observations[0].get_object_planted_uri() objects_planted = image_slice_downloader.download_raw(objects_planted_uri, view='data').split('\n') planted_objects = [] for line in objects_planted[1:]: if len(line) == 0 or line[0] == '#': continue planted_objects.append(PlantedObject(line)) false_positives_stream_writer = None matches_fptr.write("#{}\n".format(fk_candidate_observations.observations[0].rawname)) matches_fptr.write("{:1s}{} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}\n".format( "",objects_planted[0],"x_dao","y_dao","mag_dao","merr_dao", "rate_mes", "ang_mes", "dr_pixels" )) found_idxs = [] for source in fk_candidate_observations.get_sources(): reading = source.get_reading(0) third = source.get_reading(2) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) try: (x, y, mag, merr) = cutout.get_observed_magnitude() except TaskError as e: logger.warning(str(e)) mag = 0.0 merr = -1.0 matched = None for idx in range(len(planted_objects)): planted_object = planted_objects[idx] dist = math.sqrt((reading.x-planted_object.x)**2 + (reading.y - planted_object.y)**2) if matched is None or dist < matched: matched = dist matched_object_idx = idx start_jd = Time(reading.obs.header['MJD_OBS_CENTER'],format='mpc', scale='utc').jd end_jd = Time(third.obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd exptime = float(reading.obs.header['EXPTIME']) rate = math.sqrt((third.x - reading.x)**2 + (third.y - reading.y)**2)/( 24*(end_jd - start_jd) ) angle = math.degrees(math.atan2(third.y - reading.y,third.x - reading.x)) if matched > 3*rate*exptime/3600.0 and False : # this is a false positive (candidate not near artificial source) # create a .astrom style line for feeding to validate for checking later if false_positives_ftpr is None or false_positives_stream_writer is None: # create false positive file for storing results false_positives_ftpr = open(false_positive_filename,'w+') false_positives_stream_writer = StreamingAstromWriter( false_positives_ftpr,fk_candidate_observations.sys_header) false_positives_stream_writer.write_source(source) false_positives_ftpr.flush() continue elif matched_object_idx in found_idxs: repeat = '#' else: repeat = ' ' found_idxs.append(matched_object_idx) mags = [] merrs = [] for this_reading in source.get_readings()[1:]: cutout = image_slice_downloader.download_cutout(this_reading, needs_apcor=True) try: (this_x, this_y, this_mag, this_merr) = cutout.get_observed_magnitude() except TaskError as e: logger.warning(str(e)) this_mag = 0.0 this_merr = -1.0 mags.append(this_mag) merrs.append(this_merr) matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format( repeat, str(planted_objects[matched_object_idx]), reading.x, reading.y, mag, merr, rate, angle, matched)) for idx in range(len(mags)): matches_fptr.write("{:8.2f} {:8.2f}".format(mags[idx], merrs[idx])) matches_fptr.write("\n") # close the false_positives if false_positives_ftpr is not None: false_positives_ftpr.close() # record the unmatched Object.planted entries, for use in efficiency computing for idx in range(len(planted_objects)): if idx not in found_idxs: planted_object = planted_objects[idx] matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f}\n".format("",str(planted_object), 0, 0, 0, 0, 0, 0, 0)) matches_fptr.close()
""" An example of parsing an .astrom file, downloading one of the images, doing photometry calculations on it, and then displaying the image. """ from ossos import astrom from ossos import fitsviewer from ossos.downloads.cutouts import ImageCutoutDownloader sources = astrom.parse_sources("../tests/data/astromdir/realstest2.measure3.reals.astrom") reading = sources[0].get_reading(0) downloader = ImageCutoutDownloader(slice_rows=50, slice_cols=50) cutout = downloader.download_cutout(reading, needs_apcor=True) print "RA: %f" % cutout.ra print "DEC: %f" % cutout.dec print "Observed magnitude: %f" % cutout.get_observed_magnitude()[0] fitsviewer.display_hdulist(cutout.hdulist)
class CutoutDownloaderTest(FileReadingTestCase): def setUp(self): self.image_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/1584431p15.fits" self.apcor_uri = "vos://cadc.nrc.ca~vospace/OSSOS/dbimages/1584431/ccd15/1584431p15.apcor" self.reading = MagicMock(spec=SourceReading) self.reading.x = 0 self.reading.y = 0 self.reading.ra = 0 self.reading.dec = 0 self.reading.get_image_uri.return_value = self.image_uri self.reading.get_apcor_uri.return_value = self.apcor_uri self.reading.get_extension.return_value = 19 self.reading.get_original_image_size.return_value = (2000, 3000) self.reading.is_inverted.return_value = False self.needs_apcor = True self.focus = (75, 80) self.vosclient = Mock(spec=vos.Client) self.downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=50, vosclient=self.vosclient) # Mock vosclient to open a local file instead of one from vospace self.localfile = open(self.get_abs_path("data/testimg.fits"), "rb") self.apcorfile = tempfile.TemporaryFile("r+b") self.apcorfile.write("4 10 0.3 0.1") self.apcorfile.flush() self.apcorfile.seek(0) def choose_ret_val(*args, **kwargs): selected_file = None if self.image_uri in args: selected_file = self.localfile elif self.apcor_uri in args: selected_file = self.apcorfile else: self.fail("Unrecognized URI") selected_file.seek(0) return selected_file self.vosclient.open.side_effect = choose_ret_val def tearDown(self): self.localfile.close() self.apcorfile.close() def test_request_image_cutout(self): download = self.downloader.download_cutout(self.reading, focus=self.focus, needs_apcor=self.needs_apcor) assert_that(self.vosclient.open.call_args_list, contains( call(self.image_uri, view="cutout", cutout="[19][50:100,30:130]"), call(self.apcor_uri, view="data") )) # This is just a test file, make sure we can read an expected value # it. It won't have the right shape necessarily though. assert_that(download.fits_header["FILENAME"], equal_to("u5780205r_cvt.c0h"))
def recompute_mag(mpc_in): """ Get the mag of the object given the mpc.Observation """ # TODO this really shouldn't need to build a 'reading' to get the cutout... from ossos.downloads.cutouts import ImageCutoutDownloader mpc_obs = deepcopy(mpc_in) assert isinstance(mpc_obs, mpc.Observation) if mpc_obs.null_observation: return mpc_obs parts = re.search('(?P<expnum>\d{7})(?P<type>\S)(?P<ccd>\d\d)', mpc_obs.comment.frame) if parts is None: return mpc_obs expnum = parts.group('expnum') ccd = parts.group('ccd') file_type = parts.group('type') observation = astrom.Observation(expnum, file_type, ccd) assert isinstance(observation, astrom.Observation) ast_header = storage.get_astheader(int(expnum), int(ccd)) # The ZP for the current astrometric lines is the pipeline one. The new ZP is in the astheader file. new_zp = ast_header.get('PHOTZP') # The ZP in the image header is likely the one used for the original photometry. old_zp = storage.get_zeropoint(int(expnum), int(ccd)) reading = astrom.SourceReading(float(mpc_obs.comment.x), float(mpc_obs.comment.y), float(mpc_obs.comment.x), float(mpc_obs.comment.y), mpc_obs.coordinate.ra.degrees, mpc_obs.coordinate.dec.degrees, float(mpc_obs.comment.x), float(mpc_obs.comment.y), observation, ssos=True, from_input_file=True, null_observation=False, discovery=mpc_obs.discovery) # reading.is_inverted = reading.compute_inverted() reading._header = storage.get_mopheader(expnum, ccd) reading._header['MAXCOUNT'] = 30000.0 image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100) cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True) cutout.zmag = new_zp if math.fabs(new_zp - old_zp) > 0.3: logging.warning("Large change in zeropoint detected: {} -> {}".format(old_zp, new_zp)) try: (x, y, mag, merr) = cutout.get_observed_magnitude(zmag=new_zp) (x, y) = cutout.get_observed_coordinates((x, y)) except: logging.warn("Failed to do photometry.") return mpc_obs if mpc_obs.comment.mag is not None and math.fabs(mpc_obs.comment.mag - mag) > 3.5 * mpc_obs.comment.mag_uncertainty: logging.warn("recomputed magnitude shift large: {} --> {}".format(mpc_obs.mag, mag)) if math.sqrt((x - mpc_obs.comment.x) ** 2 + (y - mpc_obs.comment.y) ** 2) > 1.0: logging.warn("Centroid shifted ({},{}) -> ({},{})".format(mpc_obs.comment.x, mpc_obs.comment.y, x, y)) # Don't use the new X/Y for Hand measured entries. if str(mpc_obs.note1) != "H": mpc_obs.comment.x = x mpc_obs.comment.y = y mpc_obs.comment.mag = mag mpc_obs.comment.mag_uncertainty = merr if mpc_obs.mag is not None: mpc_obs.mag = mag return mpc_obs