def test_validate_observation():
obs = SimpleObservation('test_collection', 'test_obs_id',
Algorithm('test_name'))
validate(obs)
obs = DerivedObservation('test_collection', 'test_obs_id',
Algorithm('test_name'),
proposal=Proposal('test_proposal'),
telescope=Telescope('test_telescope'),
instrument=Instrument('test_instrument'),
target=Target('test_targets'))
obs.algorithm.keywords = 'foo'
obs.proposal.keywords = set('foo=42')
obs.telescope.keywords = set('foo:42')
obs.instrument.keywords.add("tick'marks")
obs.target.keywords = set('has multiple spaces')
test_plane = Plane('test_plane')
test_plane.provenance = Provenance('test_provenance')
test_plane.provenance.keywords.add('pipe|denied')
obs.planes['test_plane'] = test_plane
with pytest.raises(AssertionError):
validate(obs)
def _update_from_comment(observation, phangs_name, headers):
# From ER: 04-03-21
# COMMENT Produced with PHANGS-ALMA pipeline version 4.0 Build 935
# - Provenance.version
# COMMENT Galaxy properties from PHANGS sample table version 1.6
# COMMENT Calibration Level 4 (ANALYSIS_PRODUCT)
# - Calibration level (either 3 or 4)
# COMMENT PHANGS-ALMA Public Release 1
# - Provenance.project = PHANGS-ALMA
# COMMENT Generated by the Physics at High Angular resolution
# COMMENT in nearby GalaxieS (PHANGS) collaboration
# - Provenance.organization = PHANGS
# COMMENT Canonical Reference: Leroy et al. (2021), ApJ, Submitted
# - Update to reference when accepted
# COMMENT Release generated at 2021-03-04T07:28:10.245340
# - Provenance.lastExecuted
# COMMENT Data from ALMA Proposal ID: 2017.1.00886.L
# - Proposal.proposalID
# COMMENT ALMA Proposal PI: Schinnerer, Eva
# - Proposal.pi_name
# COMMENT Observed in MJD interval [58077.386275,58081.464121]
# COMMENT Observed in MJD interval [58290.770032,58365.629222]
# COMMENT Observed in MJD interval [58037.515807,58047.541173]
# COMMENT Observed in MJD interval [58353.589805,58381.654757]
# COMMENT Observed in MJD interval [58064.3677,58072.458597]
# COMMENT Observed in MJD interval [58114.347649,58139.301879]
chunk = None
for plane in observation.planes.values():
if plane.product_id != phangs_name.product_id:
continue
if plane.provenance is None:
plane.provenance = Provenance(name='PHANGS-ALMA pipeline')
for artifact in plane.artifacts.values():
if artifact.uri != phangs_name.file_uri:
continue
for part in artifact.parts.values():
chunk = part.chunks[0]
break
for entry in headers[0].get('COMMENT'):
if 'pipeline version ' in entry:
plane.provenance.version = entry.split(' version ')[1]
elif 'Calibration Level' in entry:
level = entry.split()[2]
if level == '4':
plane.calibration_level = CalibrationLevel.ANALYSIS_PRODUCT
elif 'PHANGS-ALMA Public Release' in entry:
plane.provenance.project = 'PHANGS-ALMA'
elif 'in nearby GalaxieS (PHANGS) collaboration' in entry:
plane.provenance.organization = 'PHANGS'
elif 'Release generated at ' in entry:
plane.provenance.last_executed = mc.make_time_tz(
entry.split(' at ')[1])
elif 'Data from ALMA Proposal ID:' in entry:
observation.proposal = Proposal(entry.split(':')[1].strip())
elif 'Canonical Reference: ' in entry:
plane.provenance.producer = entry.split(': ')[1]
elif 'ALMA Proposal PI:' in entry:
observation.proposal.pi_name = entry.split(': ')[1]
elif 'Observed in MJD interval ' in entry:
if chunk is not None:
bits = entry.split()[4].split(',')
start_ref_coord = RefCoord(
0.5, mc.to_float(bits[0].replace('[', '')))
end_ref_coord = RefCoord(
1.5, mc.to_float(bits[1].replace(']', '')))
sample = CoordRange1D(start_ref_coord, end_ref_coord)
if chunk.time is None:
coord_bounds = CoordBounds1D()
axis = CoordAxis1D(axis=Axis('TIME', 'd'))
chunk.time = TemporalWCS(axis, timesys='UTC')
chunk.time.axis.bounds = coord_bounds
chunk.time.axis.bounds.samples.append(sample)
def stuff(args):
obs = None
index = 0
for f_name in args.local:
product_id = args.lineage[index].split('/')[0]
t_header = Table.read(f_name, format='hdf5', path='header')
# logging.error(t_header.colnames)
# ['VERSION_MAJOR', 'VERSION_MINOR', 'TIME_IN_SEC',
# 'TIME_IN_MICROSEC', 'RUN_ID', 'ORIGIN', 'OBSMODE', 'FIELD', 'RA',
# 'DEC', 'EXPTIME', 'NUM_IMAGER']
# logging.error(t_header)
# t_header['RUN_ID'].data[0].decode() - return this string
# 20190805T024026
# logging.error(t_header['RUN_ID'].data[0].decode())
release_date = datetime.strptime(
t_header['RUN_ID'].data[0].decode(), '%Y%m%dT%H%M%S')
t_image = Table.read(f_name, format='hdf5', path='image')
# logging.error(t_image.colnames)
# ['col0', 'col1', 'col2']
# logging.error(t_image)
# logging.error(t_image['col0'].data[0])
# logging.error(t_image['col0'].data[143999])
t_catalog = Table.read(f_name, format='hdf5', path='catalog')
# logging.error(t_catalog.colnames)
# ['CAT_ID', 'GAIA_ID', '2MASS_ID', 'RA', 'DEC', 'TAOS_MAG',
# 'GAIA_MAG', '2MASS_JMAG']
# logging.error(t_catalog)
t_imager = Table.read(f_name, format='hdf5', path='imager')
# logging.error(t_imager.colnames)
# ['TEL_ID', 'CAM_ID', 'IMGR_ID', 'XLOC', 'YLOC']
# logging.error(t_imager)
t_moment = Table.read(f_name, format='hdf5', path='moment')
# logging.error(t_moment.colnames)
# ['col0', 'col1', 'col2']
# logging.error(t_moment)
t_window = Table.read(f_name, format='hdf5', path='window')
# logging.error(t_window.colnames)
# ['X0', 'X1', 'Y0', 'Y1', 'XC', 'YC']
# logging.error(t_window)
t_wcs= Table.read(f_name, format='hdf5', path='/wcs/cdmatrix')
# ['CRVAL1','CRVAL2','CRPIX1','CRPIX2','CD1_1','CD1_2','CD2_1','CD2_2']
# logging.error(t_wcs)
taos = Telescope(name='TAOS',
geo_location_x=-2354953.99637757,
geo_location_y=-4940160.3636381,
geo_location_z=3270123.70695983)
target = Target(name=str(t_header['FIELD'].data[0]),
target_type=TargetType.FIELD,
standard=None,
redshift=None,
keywords=None,
moving=None)
proposal = Proposal(id=COLLECTION,
pi_name=None,
project=COLLECTION,
title=None)
obs = SimpleObservation(collection=COLLECTION,
observation_id=args.observation[1],
sequence_number=None,
intent=ObservationIntentType.SCIENCE,
type='FIELD',
proposal=proposal,
telescope=taos,
instrument=None,
target=target,
meta_release=release_date)
provenance = Provenance(name=COLLECTION,
version='{}.{}'.format(
t_header['VERSION_MAJOR'].data[0],
t_header['VERSION_MINOR'].data[0]),
project=COLLECTION,
producer=COLLECTION,
run_id=t_header['RUN_ID'].data[0].decode(),
reference='https://taos2.asiaa.sinica.edu.tw/',
last_executed=release_date)
plane = Plane(product_id=product_id,
data_release=release_date,
meta_release=release_date,
provenance=provenance,
data_product_type=DataProductType.IMAGE,
calibration_level=CalibrationLevel.RAW_STANDARD)
artifact = mc.get_artifact_metadata(
f_name, ProductType.SCIENCE, ReleaseType.DATA,
mc.build_uri(COLLECTION, os.path.basename(f_name)))
# parts are always named '0'
part = Part('0')
# do each of the three telescopes
for telescope in [0, 1, 2]:
position = build_position(t_wcs,
t_window,
telescope)
time = build_time(t_header['TIME_IN_SEC'].data[0],
t_header['TIME_IN_MICROSEC'].data[0])
energy = build_energy()
chunk = Chunk(naxis=4,
position_axis_1=1,
position_axis_2=2,
energy_axis=3,
time_axis=4,
position=position,
energy=energy,
time=time)
part.chunks.append(chunk)
artifact.parts.add(part)
plane.artifacts.add(artifact)
obs.planes.add(plane)
index += 1
return obs
def _build_obs(override, db_content, fqn, index, almaca_name, md_name):
obs_date = db_content['Observation date'][index]
if obs_date is None:
raise mc.CadcException('No observation date for {}'.format(fqn))
else:
obs_date = time.Time(obs_date).to_datetime()
# of_site('alma')
# 2225015.30883296, -5440016.41799762, -2481631.27428014
#
size = db_content['Array'][index]
telescope = Telescope(name="ALMA-{}".format(size),
geo_location_x=2225142.18,
geo_location_y=-5440307.37,
geo_location_z=-2481029.852)
instrument = Instrument(name=_get_band_name(override))
# HK - 14-08-19
# target: this should be the science target / science fieldname and
# not the calibrator fieldname, as it is presently (i.e., should be
# J1851+0035, not J1924-2914). Or, we may need to continue to leave
# that field blank at this level. A single calibrated measurement set
# may contain multiple science target names, which would not be
# properly captured at this level. [For the raw data, the target
# field is left blank]
target = Target(name=override.get('field'),
standard=False,
moving=False,
target_type=TargetType.OBJECT)
# HK - 07-02-20
# Since we've changed what the base level observation is and can now list
# a target name in the 'Derived Observation' base plane, I believe that
# means we can also include that target's position under targetPosition.
# You've already pulled this info out for the subsequent levels of the
# hierarchy under 'position', so it's presumably fairly straightforward to
# include the same info here.
result_ra, result_dec = _get_ra_dec(md_name)
point = Point(result_ra, result_dec)
target_position = TargetPosition(
coordinates=point,
coordsys='ICRS', # from listobs output
equinox=2000.0) # a guess by google
# db_content as votable:
# >>> t.colnames
# ['Project_code', 'Source_name', 'RA', 'Dec', 'Galactic_longitude',
# 'Galactic_latitude', 'Band', 'Spatial_resolution',
# 'Frequency_resolution', 'Array', 'Mosaic', 'Integration',
# 'Release_date', 'Frequency_support', 'Velocity_resolution',
# 'Pol_products', 'Observation_date', 'PI_name', 'SB_name',
# 'Proposal_authors', 'Line_sensitivity__10_km_s_',
# 'Continuum_sensitivity', 'PWV', 'Group_ous_id', 'Member_ous_id',
# 'Asdm_uid', 'Project_title', 'Project_type', 'Scan_intent',
# 'Field_of_view', 'Largest_angular_scale', 'QA2_Status', 'COUNT',
# 'Science_keyword', 'Scientific_category', 'ASA_PROJECT_CODE']
#
# db_content.colnames as html:
#
# ['Project code', 'Source name', 'RA', 'Dec', 'Galactic longitude',
# 'Galactic latitude', 'Band', 'Spatial resolution',
# 'Frequency resolution', 'Array', 'Mosaic', 'Integration',
# 'Release date', 'Frequency support', 'Velocity resolution',
# 'Pol products', 'Observation date', 'PI name', 'SB name',
# 'Proposal authors', 'Line sensitivity (10 km/s)',
# 'Continuum sensitivity', 'PWV', 'Group ous id', 'Member ous id',
# 'Asdm uid', 'Project title', 'Project type', 'Scan intent',
# 'Field of view', 'Largest angular scale', 'QA2 Status', 'COUNT',
# 'Science keyword', 'Scientific category', 'ASA_PROJECT_CODE']
# HK - 14-08-19
# can we include the project code, and not just the observation UID
# somewhere in here? For the raw data, it looks like the project
# code was included as 'proposal: ID', whereas for the calibrated
# measurement set, proposal: ID is now set to the UID and the project
# code (2016.1.00010.S) is not captured anywhere. Could the 'project'
# field, currently set as 'null' be used for this?
proposal = Proposal(id=db_content['Project code'][index],
project=override.get('project'),
pi_name=db_content['PI name'][index],
title=db_content['Project title'][index])
keywords = db_content['Science keyword'][index]
if keywords is not None:
proposal.keywords = set(keywords.split())
environment = Environment()
environment.tau = db_content['PWV'][index] / 0.935 + 0.35
environment.wavelength_tau = 350 * units.um.to(units.meter)
intent = (ObservationIntentType.SCIENCE
if almaca_name.intent is ProductType.SCIENCE else
ObservationIntentType.CALIBRATION)
algorithm = Algorithm(name='single band split')
#
# PD, SG 15-08-19
# make it a composite, algorithm name something like
# 'target splitting'
#
observation = DerivedObservation(collection=ARCHIVE,
observation_id=almaca_name.obs_id,
sequence_number=None,
intent=intent,
type="OBJECT",
proposal=proposal,
telescope=telescope,
instrument=instrument,
target=target,
meta_release=obs_date,
algorithm=algorithm,
environment=environment,
target_position=target_position)
observation.members.add(
ObservationURI(
mc.CaomName.make_obs_uri_from_obs_id('ALMA', 'A001_X88b_X23')))
return observation