def test_continue_writing_to_existing_file(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
writer2 = StreamingAstromWriter(self.outputfile, self.sys_header)
writer2.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
def test_continue_writing_to_existing_file(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
writer2 = StreamingAstromWriter(self.outputfile, self.sys_header)
writer2.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
class StreamingAstromWriterTest(GeneralAstromWriterTest):
def setUp(self):
super(StreamingAstromWriterTest, self).setUp()
self.sys_header = {astrom.RMIN: 0.5, astrom.RMAX: 10.3,
astrom.ANGLE: -19.9, astrom.AWIDTH: 22.3}
self.writer = StreamingAstromWriter(self.outputfile, self.sys_header)
def test_write_source(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
source3 = astrom_data.sources[2]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
self.writer.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
self.writer.write_source(source3)
assert_that(self.read_output(), equal_to(get_expected(37)))
def test_continue_writing_to_existing_file(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
writer2 = StreamingAstromWriter(self.outputfile, self.sys_header)
writer2.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
class StreamingAstromWriterTest(GeneralAstromWriterTest):
def setUp(self):
super(StreamingAstromWriterTest, self).setUp()
self.sys_header = {astrom.RMIN: 0.5, astrom.RMAX: 10.3, astrom.ANGLE: -19.9, astrom.AWIDTH: 22.3}
self.writer = StreamingAstromWriter(self.outputfile, self.sys_header)
def test_write_source(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
source3 = astrom_data.sources[2]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
self.writer.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
self.writer.write_source(source3)
assert_that(self.read_output(), equal_to(get_expected(37)))
def test_continue_writing_to_existing_file(self):
with open(self.get_abs_path(TEST_FILE_1), "rb") as fh:
expected_lines = fh.readlines()
def get_expected(num_lines):
return "".join(expected_lines[:num_lines])
astrom_data = self.parse(TEST_FILE_1)
source1 = astrom_data.sources[0]
source2 = astrom_data.sources[1]
assert_that(self.read_output(), equal_to(""))
self.writer.write_source(source1)
assert_that(self.read_output(), equal_to(get_expected(28)))
writer2 = StreamingAstromWriter(self.outputfile, self.sys_header)
writer2.write_source(source2)
assert_that(self.read_output(), equal_to(get_expected(32)))
def setUp(self):
super(StreamingAstromWriterTest, self).setUp()
self.sys_header = {astrom.RMIN: 0.5, astrom.RMAX: 10.3, astrom.ANGLE: -19.9, astrom.AWIDTH: 22.3}
self.writer = StreamingAstromWriter(self.outputfile, self.sys_header)
def _create_results_writer(self, full_path, parsed_data):
output_filename = full_path.replace(tasks.get_suffix(tasks.CANDS_TASK),
tasks.get_suffix(tasks.REALS_TASK))
output_filehandle = open(output_filename, "a+b")
return StreamingAstromWriter(output_filehandle, parsed_data.sys_header)
def _create_writer(self):
filename = self.get_output_filename()
return StreamingAstromWriter(self.output_context.open(filename),
self.data.sys_header)
def match_planted(astrom_filename, match_filename, false_positive_filename):
"""
Using the astrom_filename as input get the Object.planted file from VOSpace and match
planted sources with found sources.
The Object.planted list is pulled from VOSpace based on the standard file-layout and name of the
first exposure as read from the .astrom file.
:param astrom_filename: name of the fk*reals.astrom file to check against Object.planted
:param match_filename: a file that will contain a list of all planted sources and the matched found source
:param false_positive_filename: .astrom format output containing input objects that had no match in planted
"""
image_slice_downloader = ImageCutoutDownloader(slice_rows=100, slice_cols=100)
fk_candidate_observations = astrom.parse(astrom_filename)
matches_fptr = storage.open_vos_or_local(match_filename,'w')
objects_planted_uri = fk_candidate_observations.observations[0].get_object_planted_uri()
objects_planted = image_slice_downloader.download_raw(objects_planted_uri, view='data').split('\n')
planted_objects = []
for line in objects_planted[1:]:
if len(line) == 0 or line[0] == '#':
continue
planted_objects.append(PlantedObject(line))
false_positives_stream_writer = None
matches_fptr.write("#{}\n".format(fk_candidate_observations.observations[0].rawname))
matches_fptr.write("{:1s}{} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}\n".format(
"",objects_planted[0],"x_dao","y_dao","mag_dao","merr_dao", "rate_mes", "ang_mes", "dr_pixels" ))
found_idxs = []
for source in fk_candidate_observations.get_sources():
reading = source.get_reading(0)
third = source.get_reading(2)
cutout = image_slice_downloader.download_cutout(reading, needs_apcor=True)
try:
(x, y, mag, merr) = cutout.get_observed_magnitude()
except TaskError as e:
logger.warning(str(e))
mag = 0.0
merr = -1.0
matched = None
for idx in range(len(planted_objects)):
planted_object = planted_objects[idx]
dist = math.sqrt((reading.x-planted_object.x)**2 + (reading.y - planted_object.y)**2)
if matched is None or dist < matched:
matched = dist
matched_object_idx = idx
start_jd = Time(reading.obs.header['MJD_OBS_CENTER'],format='mpc', scale='utc').jd
end_jd = Time(third.obs.header['MJD_OBS_CENTER'], format='mpc', scale='utc').jd
exptime = float(reading.obs.header['EXPTIME'])
rate = math.sqrt((third.x - reading.x)**2 + (third.y - reading.y)**2)/(
24*(end_jd - start_jd) )
angle = math.degrees(math.atan2(third.y - reading.y,third.x - reading.x))
if matched > 3*rate*exptime/3600.0 and False :
# this is a false positive (candidate not near artificial source)
# create a .astrom style line for feeding to validate for checking later
if false_positives_ftpr is None or false_positives_stream_writer is None:
# create false positive file for storing results
false_positives_ftpr = open(false_positive_filename,'w+')
false_positives_stream_writer = StreamingAstromWriter(
false_positives_ftpr,fk_candidate_observations.sys_header)
false_positives_stream_writer.write_source(source)
false_positives_ftpr.flush()
continue
elif matched_object_idx in found_idxs:
repeat = '#'
else:
repeat = ' '
found_idxs.append(matched_object_idx)
mags = []
merrs = []
for this_reading in source.get_readings()[1:]:
cutout = image_slice_downloader.download_cutout(this_reading, needs_apcor=True)
try:
(this_x, this_y, this_mag, this_merr) = cutout.get_observed_magnitude()
except TaskError as e:
logger.warning(str(e))
this_mag = 0.0
this_merr = -1.0
mags.append(this_mag)
merrs.append(this_merr)
matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} ".format(
repeat,
str(planted_objects[matched_object_idx]), reading.x, reading.y, mag, merr, rate, angle, matched))
for idx in range(len(mags)):
matches_fptr.write("{:8.2f} {:8.2f}".format(mags[idx], merrs[idx]))
matches_fptr.write("\n")
# close the false_positives
if false_positives_ftpr is not None:
false_positives_ftpr.close()
# record the unmatched Object.planted entries, for use in efficiency computing
for idx in range(len(planted_objects)):
if idx not in found_idxs:
planted_object = planted_objects[idx]
matches_fptr.write("{:1s}{} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8.2f}\n".format("",str(planted_object),
0, 0, 0, 0, 0, 0, 0))
matches_fptr.close()
def setUp(self):
super(StreamingAstromWriterTest, self).setUp()
self.sys_header = {astrom.RMIN: 0.5, astrom.RMAX: 10.3,
astrom.ANGLE: -19.9, astrom.AWIDTH: 22.3}
self.writer = StreamingAstromWriter(self.outputfile, self.sys_header)
