def build_chunk_time(chunk, header, name):
"""
:param chunk: CAOM2 Chunk instance for which to set time.
:param header: FITS header with the keywords for value extraction.
:param name: str for logging information only.
:return:
"""
logging.debug(f'Begin build_chunk_time for {name}.')
# DB 02-07-20
# time metadata comes from MJD_OBS and EXPTIME, it's not
# an axis requiring cutout support
exp_time = header.get('EXPTIME')
mjd_obs = header.get('MJD-OBS')
if exp_time is None or mjd_obs is None:
chunk.time = None
else:
if chunk.time is None:
coord_error = CoordError(syser=1e-07, rnder=1e-07)
time_axis = CoordAxis1D(axis=Axis('TIME', 'd'), error=coord_error)
chunk.time = TemporalWCS(axis=time_axis, timesys='UTC')
ref_coord = RefCoord(pix=0.5, val=mjd_obs)
chunk.time.axis.function = CoordFunction1D(
naxis=1, delta=mc.convert_to_days(exp_time), ref_coord=ref_coord)
chunk.time.exposure = exp_time
chunk.time.resolution = mc.convert_to_days(exp_time)
logging.debug(f'End build_chunk_time.')
def _augment_artifact(obs_id, artifact):
chunk = artifact.parts['0'].chunks[0]
bounds = None
exposure = None
version = None
reference = None
found = False
logging.debug(f'Scrape for time metadata for {obs_id}')
global obs_metadata
if obs_metadata is None:
obs_metadata = scrape.retrieve_obs_metadata(obs_id)
if obs_metadata is not None and len(obs_metadata) > 0:
bounds, exposure = _build_time(
obs_metadata.get('Observation Start'),
obs_metadata.get('Observation End'),
obs_metadata.get('On Source'),
)
version = obs_metadata.get('Pipeline Version')
reference = obs_metadata.get('reference')
found = True
else:
logging.warning(f'Found no time metadata for {obs_id}')
if found:
time_axis = CoordAxis1D(Axis('TIME', 'd'))
time_axis.bounds = bounds
chunk.time = TemporalWCS(time_axis)
chunk.time.exposure = exposure
chunk.time_axis = None
return version, reference
else:
return None, None
def _update_time(chunk, headers):
"""Create TemporalWCS information using FITS header information.
This information should always be available from the file."""
logging.debug('Begin _update_time.')
mc.check_param(chunk, Chunk)
mjd_start = headers[0].get('MJD_STAR')
mjd_end = headers[0].get('MJD_END')
if mjd_start is None or mjd_end is None:
mjd_start, mjd_end = ac.find_time_bounds(headers)
if mjd_start is None or mjd_end is None:
chunk.time = None
logging.debug('Cannot calculate mjd_start {} or mjd_end {}'.format(
mjd_start, mjd_end))
else:
logging.debug('Calculating range with start {} and end {}.'.format(
mjd_start, mjd_start))
start = RefCoord(0.5, mjd_start)
end = RefCoord(1.5, mjd_end)
time_cf = CoordFunction1D(1, headers[0].get('TEFF'), start)
time_axis = CoordAxis1D(Axis('TIME', 'd'), function=time_cf)
time_axis.range = CoordRange1D(start, end)
chunk.time = TemporalWCS(time_axis)
chunk.time.exposure = headers[0].get('TEFF')
chunk.time.resolution = 0.1
chunk.time.timesys = 'UTC'
chunk.time.trefpos = 'TOPOCENTER'
chunk.time_axis = 4
logging.debug('Done _update_time.')
def build_temporal_wcs_append_sample(temporal_wcs, lower, upper):
"""All the CAOM entities for building a TemporalWCS instance with
a bounds definition, or appending a sample, in one function.
"""
if temporal_wcs is None:
samples = TypedList(CoordRange1D, )
bounds = CoordBounds1D(samples=samples)
temporal_wcs = TemporalWCS(axis=CoordAxis1D(axis=Axis('TIME', 'd'),
bounds=bounds),
timesys='UTC')
start_ref_coord = RefCoord(pix=0.5, val=lower)
end_ref_coord = RefCoord(pix=1.5, val=upper)
sample = CoordRange1D(start_ref_coord, end_ref_coord)
temporal_wcs.axis.bounds.samples.append(sample)
return temporal_wcs
def _augment_artifact(obs_id, artifact, csv_file):
chunk = artifact.parts['0'].chunks[0]
bounds = None
exposure = None
version = None
reference = None
for ii in csv_file:
if obs_id in ii:
bounds, exposure = _build_time(ii)
version = ii[1].strip()
reference = ii[2].strip()
break
time_axis = CoordAxis1D(Axis('TIME', 'd'))
time_axis.bounds = bounds
chunk.time = TemporalWCS(time_axis)
chunk.time.exposure = exposure
return version, reference
def build_time(seconds, microseconds):
mjd_start = AstroTime(seconds, format='unix')
mjd_start.format = 'mjd'
start = RefCoord(0.5, mjd_start.value)
end_ms = seconds + (microseconds / 1e7)
mjd_end = AstroTime(end_ms, format='unix')
mjd_end.format = 'mjd'
end = RefCoord(1.5, mjd_end.value)
axis = CoordAxis1D(axis=Axis(ctype='TIME', cunit='d'))
axis.range = CoordRange1D(start, end)
return TemporalWCS(axis=axis,
timesys='UTC',
trefpos=None,
mjdref=None,
exposure=(microseconds / 1e7),
resolution=None)
def _update_ngvs_time(chunk, provenance, obs_id):
logging.debug(f'Begin _update_ngvs_time for {obs_id}')
if (chunk is not None and provenance is not None and
len(provenance.inputs) > 0):
# bounds = ctor
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
client = CAOM2RepoClient(
subject, config.logging_level, 'ivo://cadc.nrc.ca/ams')
metrics = mc.Metrics(config)
bounds = CoordBounds1D()
min_date = 0
max_date = sys.float_info.max
exposure = 0
for entry in provenance.inputs:
ip_obs_id, ip_product_id = mc.CaomName.decompose_provenance_input(
entry.uri)
logging.info(f'Retrieving provenance metadata for {ip_obs_id}.')
ip_obs = mc.repo_get(client, 'CFHT', ip_obs_id, metrics)
if ip_obs is not None:
ip_plane = ip_obs.planes.get(ip_product_id)
if (ip_plane is not None and ip_plane.time is not None and
ip_plane.time.bounds is not None):
bounds.samples.append(CoordRange1D(
RefCoord(pix=0.5, val=ip_plane.time.bounds.lower),
RefCoord(pix=1.5, val=ip_plane.time.bounds.upper)))
min_date = min(ip_plane.time.bounds.lower, min_date)
max_date = max(ip_plane.time.bounds.upper, max_date)
exposure += ip_plane.time.exposure
axis = Axis(ctype='TIME', cunit='d')
time_axis = CoordAxis1D(axis=axis,
error=None,
range=None,
bounds=bounds,
function=None)
temporal_wcs = TemporalWCS(axis=time_axis, timesys=None, trefpos=None,
mjdref=None, exposure=mc.to_float(exposure),
resolution=None)
chunk.time = temporal_wcs
logging.debug(f'End _update_ngvs_time.')
def _update_time(part, chunk, header, obs_id):
logging.debug(f'Begin _update_time for {obs_id} part {part.name}.')
# DB 02-07-20
# time metadata comes from MJD_OBS and EXPTIME, it's not
# an axis requiring cutout support
exp_time = header.get('EXPTIME')
mjd_obs = header.get('MJD_OBS')
if exp_time is None or mjd_obs is None:
chunk.time = None
else:
if chunk.time is None:
coord_error = CoordError(syser=1e-07, rnder=1e-07)
time_axis = CoordAxis1D(axis=Axis('TIME', 'd'), error=coord_error)
chunk.time = TemporalWCS(axis=time_axis, timesys='UTC')
ref_coord = RefCoord(pix=0.5, val=mjd_obs)
chunk.time.axis.function = CoordFunction1D(
naxis=1, delta=mc.convert_to_days(exp_time), ref_coord=ref_coord)
chunk.time.exposure = float(exp_time)
chunk.time.resolution = mc.convert_to_days(exp_time)
logging.debug(f'End _update_time.')
def _update_time(self, chunk, obs_id):
"""Create TemporalWCS information using FITS header information.
This information should always be available from the file."""
self._logger.debug('Begin _update_time.')
mc.check_param(chunk, Chunk)
mjd_start = self._headers[0].get('MJD_STAR')
mjd_end = self._headers[0].get('MJD_END')
if mjd_start is None or mjd_end is None:
mjd_start, mjd_end = ac.find_time_bounds(self._headers)
if mjd_start is None or mjd_end is None:
chunk.time = None
self._logger.debug(
f'Cannot calculate MJD_STAR {mjd_start} or ' f'MDJ_END'
f' {mjd_end}'
)
elif mjd_start == 'NaN' or mjd_end == 'NaN':
raise mc.CadcException(
f'Invalid time values MJD_STAR {mjd_start} or MJD_END '
f'{mjd_end} for {obs_id}, stopping ingestion.'
)
else:
self._logger.debug(
f'Calculating range with start {mjd_start} and end {mjd_end}.'
)
start = RefCoord(0.5, mjd_start)
end = RefCoord(1.5, mjd_end)
time_cf = CoordFunction1D(1, self._headers[0].get('TEFF'), start)
time_axis = CoordAxis1D(Axis('TIME', 'd'), function=time_cf)
time_axis.range = CoordRange1D(start, end)
chunk.time = TemporalWCS(time_axis)
chunk.time.exposure = self._headers[0].get('TEFF')
chunk.time.resolution = 0.1
chunk.time.timesys = 'UTC'
chunk.time.trefpos = 'TOPOCENTER'
chunk.time_axis = None
self._logger.debug('Done _update_time.')
def test_augment_artifact_bounds_range_from_blueprint():
test_blueprint = ObsBlueprint(energy_axis=1,
time_axis=2,
polarization_axis=3,
position_axes=(4, 5))
test_blueprint.set('Chunk.energy.axis.range.start.pix', '145.0')
test_blueprint.set('Chunk.energy.axis.range.start.val', '-60000.0')
test_blueprint.set('Chunk.energy.axis.range.end.pix', '-824.46002')
test_blueprint.set('Chunk.energy.axis.range.end.val', '1')
test_blueprint.set('Chunk.time.axis.range.start.pix', '145.0')
test_blueprint.set('Chunk.time.axis.range.start.val', '-60000.0')
test_blueprint.set('Chunk.time.axis.range.end.pix', '-824.46002')
test_blueprint.set('Chunk.time.axis.range.end.val', '1')
test_blueprint.set('Chunk.polarization.axis.range.start.pix', '145.0')
test_blueprint.set('Chunk.polarization.axis.range.start.val', '-60000.0')
test_blueprint.set('Chunk.polarization.axis.range.end.pix', '-824.46002')
test_blueprint.set('Chunk.polarization.axis.range.end.val', '1')
test_blueprint.set('Chunk.position.axis.range.start.coord1.pix', '145.0')
test_blueprint.set('Chunk.position.axis.range.start.coord1.val',
'-60000.0')
test_blueprint.set('Chunk.position.axis.range.end.coord1.pix',
'-824.46002')
test_blueprint.set('Chunk.position.axis.range.end.coord1.val', '1')
test_blueprint.set('Chunk.position.axis.range.start.coord2.pix', '145.0')
test_blueprint.set('Chunk.position.axis.range.start.coord2.val',
'-60000.0')
test_blueprint.set('Chunk.position.axis.range.end.coord2.pix',
'-824.46002')
test_blueprint.set('Chunk.position.axis.range.end.coord2.val', '1')
test_fitsparser = FitsParser(sample_file_4axes,
test_blueprint,
uri='ad:TEST/test_blueprint')
test_chunk = Chunk()
test_chunk.energy = SpectralWCS(CoordAxis1D(Axis('WAVE', 'm')), 'TOPOCENT')
test_chunk.time = TemporalWCS(CoordAxis1D(Axis('TIME', 'd')))
test_chunk.polarization = PolarizationWCS(CoordAxis1D(Axis('STOKES')))
test_chunk.position = SpatialWCS(
CoordAxis2D(Axis('RA', 'deg'), Axis('DEC', 'deg')))
test_fitsparser._try_range_with_blueprint(test_chunk, 0)
assert test_chunk.energy.axis.range is not None, \
'chunk.energy.axis.range should be declared'
assert test_chunk.time.axis.range is not None, \
'chunk.time.axis.range should be declared'
assert test_chunk.polarization.axis.range is not None, \
'chunk.polarization.axis.range should be declared'
assert test_chunk.position.axis.range is not None, \
'chunk.position.axis.range should be declared'
ex = _get_from_str_xml(EXPECTED_ENERGY_RANGE_BOUNDS_XML,
ObservationReader()._get_spectral_wcs, 'energy')
assert ex is not None, \
'energy string from expected output should be declared'
result = get_differences(ex, test_chunk.energy)
assert result is None
ex = _get_from_str_xml(EXPECTED_TIME_RANGE_BOUNDS_XML,
ObservationReader()._get_temporal_wcs, 'time')
assert ex is not None, \
'time string from expected output should be declared'
result = get_differences(ex, test_chunk.time)
assert result is None
ex = _get_from_str_xml(EXPECTED_POL_RANGE_BOUNDS_XML,
ObservationReader()._get_polarization_wcs,
'polarization')
assert ex is not None, \
'polarization string from expected output should be declared'
result = get_differences(ex, test_chunk.polarization)
assert result is None
ex = _get_from_str_xml(EXPECTED_POS_RANGE_BOUNDS_XML,
ObservationReader()._get_spatial_wcs, 'position')
assert ex is not None, \
'position string from expected output should be declared'
result = get_differences(ex, test_chunk.position)
assert result is None
def _update_from_comment(observation, phangs_name, headers):
# From ER: 04-03-21
# COMMENT Produced with PHANGS-ALMA pipeline version 4.0 Build 935
# - Provenance.version
# COMMENT Galaxy properties from PHANGS sample table version 1.6
# COMMENT Calibration Level 4 (ANALYSIS_PRODUCT)
# - Calibration level (either 3 or 4)
# COMMENT PHANGS-ALMA Public Release 1
# - Provenance.project = PHANGS-ALMA
# COMMENT Generated by the Physics at High Angular resolution
# COMMENT in nearby GalaxieS (PHANGS) collaboration
# - Provenance.organization = PHANGS
# COMMENT Canonical Reference: Leroy et al. (2021), ApJ, Submitted
# - Update to reference when accepted
# COMMENT Release generated at 2021-03-04T07:28:10.245340
# - Provenance.lastExecuted
# COMMENT Data from ALMA Proposal ID: 2017.1.00886.L
# - Proposal.proposalID
# COMMENT ALMA Proposal PI: Schinnerer, Eva
# - Proposal.pi_name
# COMMENT Observed in MJD interval [58077.386275,58081.464121]
# COMMENT Observed in MJD interval [58290.770032,58365.629222]
# COMMENT Observed in MJD interval [58037.515807,58047.541173]
# COMMENT Observed in MJD interval [58353.589805,58381.654757]
# COMMENT Observed in MJD interval [58064.3677,58072.458597]
# COMMENT Observed in MJD interval [58114.347649,58139.301879]
chunk = None
for plane in observation.planes.values():
if plane.product_id != phangs_name.product_id:
continue
if plane.provenance is None:
plane.provenance = Provenance(name='PHANGS-ALMA pipeline')
for artifact in plane.artifacts.values():
if artifact.uri != phangs_name.file_uri:
continue
for part in artifact.parts.values():
chunk = part.chunks[0]
break
for entry in headers[0].get('COMMENT'):
if 'pipeline version ' in entry:
plane.provenance.version = entry.split(' version ')[1]
elif 'Calibration Level' in entry:
level = entry.split()[2]
if level == '4':
plane.calibration_level = CalibrationLevel.ANALYSIS_PRODUCT
elif 'PHANGS-ALMA Public Release' in entry:
plane.provenance.project = 'PHANGS-ALMA'
elif 'in nearby GalaxieS (PHANGS) collaboration' in entry:
plane.provenance.organization = 'PHANGS'
elif 'Release generated at ' in entry:
plane.provenance.last_executed = mc.make_time_tz(
entry.split(' at ')[1])
elif 'Data from ALMA Proposal ID:' in entry:
observation.proposal = Proposal(entry.split(':')[1].strip())
elif 'Canonical Reference: ' in entry:
plane.provenance.producer = entry.split(': ')[1]
elif 'ALMA Proposal PI:' in entry:
observation.proposal.pi_name = entry.split(': ')[1]
elif 'Observed in MJD interval ' in entry:
if chunk is not None:
bits = entry.split()[4].split(',')
start_ref_coord = RefCoord(
0.5, mc.to_float(bits[0].replace('[', '')))
end_ref_coord = RefCoord(
1.5, mc.to_float(bits[1].replace(']', '')))
sample = CoordRange1D(start_ref_coord, end_ref_coord)
if chunk.time is None:
coord_bounds = CoordBounds1D()
axis = CoordAxis1D(axis=Axis('TIME', 'd'))
chunk.time = TemporalWCS(axis, timesys='UTC')
chunk.time.axis.bounds = coord_bounds
chunk.time.axis.bounds.samples.append(sample)