def _update_spatial_wcs(part, chunk, headers, obs_id): logging.debug(f'Begin _update_spatial_wcs for {obs_id}') if part.name == '1': # DB/NC 22-07-20 # # Use the NAXIS* values from the 'SCI' extension. # # CD*: That should give approximately correct pixel scales along the # axes in degrees/pixel. # # The hard coded values for CRPIX assume the RA/DEC values are at the # centre of the 60/62 array of pixels in the final product. The # primary header CRPIX values would have referred to the original # ~2000 x ~2000 pixel detector array I’m guessing. header = headers[0] idx = mc.to_int(part.name) header['NAXIS1'] = headers[idx].get('NAXIS1') header['NAXIS2'] = headers[idx].get('NAXIS2') header['CD1_1'] = 3.0 / (header.get('NAXIS1') * 3600.0) header['CD2_2'] = 3.0 / (header.get('NAXIS2') * 3600.0) header['CD1_2'] = 0.0 header['CD2_1'] = 0.0 header['CRPIX1'] = header.get('NAXIS1') / 2.0 header['CRPIX2'] = header.get('NAXIS2') / 2.0 wcs_parser = WcsParser(header, obs_id, 0) wcs_parser.augment_position(chunk) chunk.position_axis_1 = 1 chunk.position_axis_2 = 2 logging.debug(f'End _update_spatial_wcs')
def get_version(entry): """The parameter may be a URI, or just the file name.""" # file name looks like: # 'VLASS1.2.ql.T20t12.J092604+383000.10.2048.v2.I.iter1.image. # 'pbcor.tt0.rms.subim.fits' file_name = entry if '/' in entry: file_name = mc.CaomName(entry).file_name bits = file_name.split('.') version_str = bits[7].replace('v', '') return mc.to_int(version_str)
def _try_to_append_to_cache(self, run_id): if run_id is None or run_id == '': # the case of no value for run_id return sem = CFHTCache.semester(run_id) sem_int = mc.to_int(sem[:-1]) updated_content = False if len(sem) < 3 or not sem[0] in ['0', '1', '2'] or sem_int < 9: # the URL here only works from 2009B on return base_url = 'http://www.cfht.hawaii.edu/en/science/QSO/' semester_url = f'{base_url}20{sem}/' self._logger.info( f'Checking for semester information at {semester_url}') response = mc.query_endpoint(semester_url) soup = BeautifulSoup(response.text, features='lxml') response.close() html_table = soup.find('table') rows = html_table.find_all('a', string=re.compile('\\.html')) for row in rows: inst_url = row.get('href') if not isinstance(inst_url, str) and 'qso_prog_' not in inst_url: continue if not inst_url.startswith('http'): inst_url = f'{semester_url}/{inst_url}' self._logger.info( f'Querying {inst_url} for new project information.') inst_response = mc.query_endpoint(inst_url) inst_soup = BeautifulSoup(inst_response.text, features='lxml') inst_response.close() table_rows = inst_soup.find_all('tr') for table_row in table_rows: tds = table_row.find_all('td') count = 0 for td in tds: if count == 0: program_id = td.text if count == 5: title = td.text updated_content = True self._project_titles[program_id] = \ CFHTCache.clean(title) break count += 1 if updated_content: self.save() self._cached_semesters.append(CFHTCache.semester(run_id))
def _rebin_factor(self, a, new_shape): """ Re-bin an array to a new shape. :param new_shape must be a factor of a.shape. """ assert len(a.shape) == len(new_shape) assert not np.sometrue(np.mod(a.shape, new_shape)) slices = [ slice(None, None, mc.to_int(old / new)) for old, new in zip(a.shape, new_shape) ] self._logger.debug(slices) return a[slices]
def update(observation, **kwargs): """Called to fill multiple CAOM model elements and/or attributes, must have this signature for import_module loading and execution. :param observation A CAOM Observation model instance. :param **kwargs Everything else.""" logging.debug('Begin update.') mc.check_param(observation, Observation) headers = None if 'headers' in kwargs: headers = kwargs['headers'] fqn = None if 'fqn' in kwargs: fqn = kwargs['fqn'] try: for plane in observation.planes.values(): for artifact in plane.artifacts.values(): temp_parts = TypedOrderedDict(Part, ) # need to rename the BINARY TABLE extensions, which have # differently telemetry, and remove their chunks for part_key in ['1', '2', '3', '4', '5']: if part_key in artifact.parts: hdu_count = mc.to_int(part_key) temp = artifact.parts.pop(part_key) temp.product_type = ProductType.AUXILIARY temp.name = headers[hdu_count].get('EXTNAME') while len(temp.chunks) > 0: temp.chunks.pop() temp_parts.add(temp) for part in artifact.parts.values(): if part.name == '0': part.product_type = artifact.product_type for chunk in part.chunks: chunk.product_type = artifact.product_type _build_chunk_energy(chunk, headers) _build_chunk_position(chunk, headers, observation.observation_id) chunk.time_axis = None for part in temp_parts.values(): artifact.parts.add(part) logging.debug('Done update.') except Exception as e: logging.error(e) logging.debug(traceback.format_exc()) observation = None return observation
def rename_parts(observation, headers): """ By default, the values for part.name are extension numbers. Replace those with the value of the EXTNAME keyword. The part.name is the key value in the TypedOrderedDict, so this is done as a pop/push. :param observation Observation instance with parts that may have the wrong names :param headers astropy FITS Header list """ part_keys = [str(ii) for ii in range(1, headers[0].get('NEXTEND') + 1)] for plane in observation.planes.values(): for artifact in plane.artifacts.values(): temp_parts = TypedOrderedDict(Part, ) for part_key in part_keys: if part_key in artifact.parts: hdu_count = mc.to_int(part_key) temp = artifact.parts.pop(part_key) temp.name = headers[hdu_count].get('EXTNAME') temp_parts.add(temp) for part in temp_parts.values(): artifact.parts.add(part)
def get_position_axis_function_naxis2(header): result = mc.to_int(header.get('NAXIS2')) if result is not None: result = result / 2.0 return result
def _do_sci(self, hdu_list, header, storage_name, science_fqn, preview_fqn, thumb_fqn): logging.debug(f'Do science preview augmentation with {science_fqn}') count = 0 detector = header.get('DETECTOR') instrument = header.get('INSTRUME') if detector in [ 'SITe-4', 'UBC-1', 'SITe-2', 'SITe-5', 'E2V-1', 'E2V-4' ]: # unprocessed CCD data if detector == 'SITe-4': axis = 'NAXIS2' naxis1 = mc.to_int(header.get(axis)) xc = naxis1 / 2 xs = 512 xoffset = xc - xs / 2 rotate = '90.0' geometry = '256x' + str(xs) + '+1+' + str(xoffset) resize1 = 'x1024' resize2 = 'x256' else: axis = 'NAXIS1' naxis1 = mc.to_int(header.get(axis)) xc = naxis1 / 2 xs = 512 xoffset = xc - xs / 2 rotate = '0.0' geometry = str(xs) + 'x256+' + str(xoffset) + '+1' resize1 = '1024x1024' resize2 = '256' if 'Imager' in instrument: mc.exec_cmd(f'convert -resize 1024x1024 ' f'-normalize -negate {science_fqn} {preview_fqn}') mc.exec_cmd(f'convert -resize 256x256 -normalize -negate ' f'{science_fqn} {thumb_fqn}') else: mc.exec_cmd(f'convert -resize {resize1} -rotate {rotate} ' f'-normalize -negate {science_fqn} {preview_fqn}') mc.exec_cmd(f'convert -crop {geometry} -resize {resize2} ' f'-rotate {rotate} -normalize ' f'-negate {science_fqn} {thumb_fqn}') count = 2 else: # unprocessed RETICON spectrum object_type = header.get('OBJECT') if object_type is not None: naxis1 = header.get('NAXIS1') logging.info(f'Object: {object_type}') signal = hdu_list[0].data[0] baseline = hdu_list[0].data[1] flux = np.subtract(signal, baseline) wl = [] for i in range(0, naxis1): wl.append(i + 1) wln = np.array(wl) self._write_files_to_disk( wln, flux, 'Pixel', f'{storage_name.file_id}: {object_type}', thumb_fqn, preview_fqn) count = 2 return count
def update(observation, **kwargs): """Called to fill multiple CAOM model elements and/or attributes (an n:n relationship between TDM attributes and CAOM attributes). Must have this signature for import_module loading and execution. :param observation A CAOM Observation model instance. :param **kwargs Everything else.""" logging.debug('Begin update.') mc.check_param(observation, Observation) headers = kwargs.get('headers') fqn = kwargs.get('fqn') uri = kwargs.get('uri') gem_proc_name = None # ok not to use builder here, since the obs_id value is never used later if uri is not None: temp = mc.CaomName(uri).file_name gem_proc_name = builder.GemProcName(entry=temp) if fqn is not None: gem_proc_name = builder.GemProcName(entry=fqn) if gem_proc_name is None: raise mc.CadcException(f'Need one of fqn or uri defined for ' f'{observation.observation_id}') for plane in observation.planes.values(): if plane.product_id != gem_proc_name.product_id: continue for artifact in plane.artifacts.values(): for part in artifact.parts.values(): idx = mc.to_int(part.name) header = headers[idx] extname = header.get('EXTNAME') # DB 22-07-20 # There are a few other EXTNAME values to look at for # part.ProductType. MDF values would be ‘AUXILIARY’. The # ones currently called “CAL” are likely best set to ‘INFO’ # since it contains info about datasets used to produce the # product. # # DB 07-08-20 # EXTNAME in (‘DQ’, ‘VAR’) should both have # ProductType.NOISE. ‘CAL’ should no longer exist - it’s now # BPM. Default type is 'AUXILIARY', 'SCI' is type 'SCIENCE' if extname == 'SCI': part.product_type = ProductType.SCIENCE elif extname in ['DQ', 'VAR']: part.product_type = ProductType.NOISE else: part.product_type = ProductType.AUXILIARY if part.product_type in [ ProductType.SCIENCE, ProductType.INFO, ]: for chunk in part.chunks: filter_name = headers[0].get('FILTER').split('_')[0] _update_energy( chunk, headers[idx], filter_name, observation.observation_id, ) _update_time(part, chunk, headers[0], observation.observation_id) if part.product_type == ProductType.SCIENCE: _update_spatial_wcs( part, chunk, headers, observation.observation_id, ) chunk.naxis = header.get('NAXIS') if (chunk.position is None and chunk.naxis is not None): chunk.naxis = None if (chunk.time is not None and chunk.time.axis is not None and chunk.time.axis.function is not None and chunk.time.axis.function.delta == 1.0): # these are the default values, and they make # the time range start in 1858 chunk.time = None else: # DB 21-07-20 # ignore WCS information unless product type == SCIENCE while len(part.chunks) > 0: del part.chunks[-1] if (observation.proposal is not None and observation.proposal.id is not None and observation.proposal.pi_name is None): program = program_metadata.get_pi_metadata(observation.proposal.id) if program is not None: observation.proposal.pi_name = program.get('pi_name') observation.proposal.title = program.get('title') if isinstance(observation, SimpleObservation): # undo the observation-level metadata modifications for updated # Gemini records observation.algorithm = Algorithm(name='exposure') else: cc.update_observation_members(observation) logging.debug('Done update.') return observation