def test_masked_as_array_with_mixin(): """Test that as_array() and Table.mask attr work with masked mixin columns""" t = Table() t['a'] = Time([1, 2], format='cxcsec') t['b'] = [3, 4] t['c'] = [5, 6] * u.m # With no mask, the output should be ndarray ta = t.as_array() assert isinstance(ta, np.ndarray) and not isinstance(ta, np.ma.MaskedArray) # With a mask, output is MaskedArray t['a'][1] = np.ma.masked ta = t.as_array() assert isinstance(ta, np.ma.MaskedArray) assert np.all(ta['a'].mask == [False, True]) assert np.isclose(ta['a'][0].cxcsec, 1.0) assert np.all(ta['b'].mask == False) # noqa assert np.all(ta['c'].mask == False) # noqa # Check table ``mask`` property tm = t.mask assert np.all(tm['a'] == [False, True]) assert np.all(tm['b'] == False) # noqa assert np.all(tm['c'] == False) # noqa
def test_write_bintable(self): '''test write_bintable''' from ..io.util import write_bintable, fitsheader hdr = fitsheader(dict(A=1, B=2)) hdr['C'] = ('BLAT', 'FOO') data = Table() data['X'] = [1, 2, 3] data['Y'] = [3, 4, 5] write_bintable(self.testfile, data, header=hdr) result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(result.dtype.names, data.dtype.names) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) #- repeat with other data types os.remove(self.testfile) hdr = dict(A=1, B=2) data = data.as_array() write_bintable(self.testfile, data, header=hdr) result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(result.dtype.names, data.dtype.names) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) for key in hdr.keys(): self.assertIn(key, newhdr)
def identify_stars(x, y, wcs, identify_catalog, science_catalog=None, filter=None, limit_angle='2 arcsec'): """Identify stars with coordinates x and y in a wcs frame and return pertinent parameters from the catalogs.""" cat = identify_catalog ra, dec = wcs_xy2radec(x, y, wcs) cat_res = cat.match_objects(ra, dec, filter, limit_angle=limit_angle) name = cat_res['id'] mag = cat_res['flux'] mag_err = cat_res['flux_error'] res = Table() if science_catalog is not None: sci = science_catalog sci_res = sci.match_objects(ra, dec, limit_angle=limit_angle) res['sci_id'] = process_list(string_fix, sci_res['id']) res['cat_id'] = process_list(string_fix, name) res['ra'] = ra res['dec'] = dec res['cat_mag'] = mag res['cat_mag_err'] = mag_err return res.as_array()
def readMatch(self, fname, pixId): names=['ext_shapeHSM_HsmShapeRegauss_e1','ext_shapeHSM_HsmShapeRegauss_e2','base_SdssShape_x','base_SdssShape_y',\ 'modelfit_CModel_instFlux','modelfit_CModel_instFluxErr','ext_shapeHSM_HsmShapeRegauss_resolution'] pltDir = '../../galSim-HSC/s19/s19-1/anaCat_newS19Mask_fdeltacut/plot/optimize_weight/' # Match pix_scale = 0.168 / 3600. cosmo252 = imgSimutil.cosmoHSTGal('252') cosmo252E = imgSimutil.cosmoHSTGal('252E') info = cosmo252.hpInfo[cosmo252.hpInfo['pix'] == pixId] nx = int(info['dra'] / pix_scale) ny = int(info['ddec'] / pix_scale) if 'CosmoE' in fname: hstcat = pyfits.getdata('hstcatE-dis4.fits') elif 'CosmoR' in fname: hstcat = pyfits.getdata('hstcatR-dis4.fits') msk = (hstcat['xI'] > 32) & (hstcat['yI'] > 32) & ( hstcat['xI'] < nx - 32) & (hstcat['yI'] < ny - 32) hstcat = hstcat[msk] xyRef = np.vstack([hstcat['xI'], hstcat['yI']]).T tree = scipy.spatial.cKDTree(xyRef) del msk, xyRef dd = self.readFits(fname) xyDat = np.vstack([dd['base_SdssShape_x'], dd['base_SdssShape_y']]).T dis, inds = tree.query(xyDat, k=1) mask = (dis <= (0.85 / 0.168)) dis = dis[mask] inds = inds[mask] dd = dd[mask] wlmsk = (catutil.get_imag(dd) < 24.5) & \ (catutil.get_abs_ellip(dd) <= 2.) & \ (catutil.get_res(dd) >= 0.3) & \ (catutil.get_snr(dd) >= 10.) & \ (catutil.get_imag_A10(dd)<25.5) & \ (catutil.get_logb(dd)<= -0.38) dd = dd[wlmsk] inds = inds[wlmsk] matcat = hstcat[inds] del mask, inds, dis dd = dd[names] dd = Table(dd.as_array(), names=names) sigmae = catutil.get_sigma_e_model(dd, pltDir) erms = catutil.get_erms_model(dd, pltDir) dd['i_hsmshaperegauss_derived_weight'] = 1. / (sigmae**2 + erms**2) #dd['i_hsmshaperegauss_derived_sigma_e']= sigmae #dd['i_hsmshaperegauss_derived_rms_e'] = erms dd['zphot'] = matcat['zphot'] dd['mag_auto'] = matcat['mag_auto'] del xyDat, wlmsk, matcat, hstcat, sigmae, erms gc.collect() return dd
def test_read_returns_image(tmpdir): # Test if CCData.read returns a image when reading a fits file containing # a table and image, in that order. tbl = Table(np.ones(10).reshape(5, 2)) img = np.ones((5, 5)) hdul = fits.HDUList(hdus=[fits.PrimaryHDU(), fits.TableHDU(tbl.as_array()), fits.ImageHDU(img)]) filename = tmpdir.join('table_image.fits').strpath hdul.writeto(filename) ccd = CCDData.read(filename, unit='adu') # Expecting to get (5, 5), the size of the image assert ccd.data.shape == (5, 5)
def test_read_returns_image(tmpdir): # Test if CCData.read returns a image when reading a fits file containing # a table and image, in that order. tbl = Table(np.ones(10).reshape(5, 2)) img = np.ones((5, 5)) hdul = fits.HDUList(hdus=[fits.PrimaryHDU(), fits.TableHDU(tbl.as_array()), fits.ImageHDU(img)]) filename = tmpdir.join('table_image.fits').strpath hdul.writeto(filename) ccd = CCDData.read(filename, unit='adu') # Expecting to get (5, 5), the size of the image assert ccd.data.shape == (5, 5)
def convert_to_hdulist(ds): """ transform TheCannon dataset into fits HDU list """ print('@Bo Zhang: ---------------------------------------------------') print('@Bo Zhang: transforming TheCannon data cubes into HDU list ...') print('@Bo Zhang: ---------------------------------------------------') # initialize data list data_list = [ ds.wl, ds.ranges, ds.contmask * 1, ds.tr_ID, ds.tr_SNR, ds.tr_flux, ds.tr_ivar, ds.tr_label, ds.test_ID, ds.test_SNR, ds.test_flux, ds.test_ivar ] name_list = [ 'wl', 'ranges', 'contmask', 'tr_ID', 'tr_SNR', 'tr_flux', 'tr_ivar', 'tr_label', 'test_ID', 'test_SNR', 'test_flux', 'test_ivar' ] hdu_format_list_rw = [ 'image', 'image', 'image', 'table', 'image', 'image', 'image', 'image', 'table', 'image', 'image', 'image' ] # construct Primary header header = fits.Header() header['author'] = 'Bo Zhang (@NAOC)' header['version'] = 'v1.0' header['last_modified'] = 'Sat 11 Jun 2016' header['data'] = 'TheCannon dataset object' # initialize HDU list print('@Bo Zhang: initializing the HDU list ...') hl = [fits.hdu.PrimaryHDU(header=header)] # construct HDU list assert len(data_list) == len(name_list) n_hdus = len(data_list) for i in xrange(n_hdus): print('@Bo Zhang: transforming HDU [%d/%d]: %s ...' % (i + 1, n_hdus, name_list[i])) if hdu_format_list_rw[i] == 'table': data = Table(data=data_list[i].reshape((-1, 1)), names=[name_list[i]]) hl.append(fits.BinTableHDU(data.as_array())) else: hl.append( fits.ImageHDU(data=np.array(data_list[i]), name=name_list[i])) print('@Bo Zhang: ---------------------------------------------------') return fits.HDUList(hl)
def match_pairs(x, y, dx, dy, tolerance=1.0, logger=logger): """Match the pairs of ordinary/extraordinary points (x, y).""" kd = cKDTree(list(zip(x, y))) px = np.array(x-dx) py = np.array(y-dy) d, ind = kd.query(list(zip(px, py)), k=1, distance_upper_bound=tolerance, n_jobs=-1) o = np.arange(len(x))[np.where(d <= tolerance)] e = np.array(ind[np.where(d <= tolerance)]) result = Table() result['o'] = o result['e'] = e return result.as_array()
def ZTF_CXC_xmatch(archive, ztfobjects, tcresults, n_results): # Filtering ztfobjects with xmatch results tcresults_byoid = tcresults.group_by('oid') keys = tcresults_byoid.groups.keys['oid'].data ztfobjects.add_index('oid') maskedztf = Table(ztfobjects.loc[keys]) # Stipulation if there is only one ZTF object that's been matched with CXC data if len(keys) == 1: maskedztf = Table(np.repeat(maskedztf, n_results[0])) n_results = np.repeat(n_results, n_results[0]) maskedztf.add_column(n_results, name='n_results') # Initializing new data columns for tap service and cone search results prefix = {'CDA': 'cda_%s', 'CSC': 'csc_%s'}[archive] for colname in tcresults.colnames: maskedztf[prefix % (colname)] = np.full(len(maskedztf), None) # Formatting filtered ztf object array mztf = maskedztf.as_array() ztf_fr = [] if len(keys) > 1: # Adding extra rows, according to the number of cone search results each ZTF object has ztf_fr = mztf ztf_nres = n_results[n_results != 0] ztf_fr = Table(np.repeat(ztf_fr, n_results)) else: ztf_fr = Table(mztf) # Sort both ztfobjects table and tap service/cone search results table by ZTF oid # so that we can easily assign columns if len(tcresults) > 1: ztf_fr.sort('oid') tcresults.sort('oid') # Filling new data columns in ztfobjects table for colname in tcresults.colnames: ztf_fr[prefix % (colname)] = tcresults[colname] ztf_fr.remove_column(prefix % ('oid')) return ztf_fr
def get_stars(self): """ Returns a list of IStar objects from the HDUL's TableHDU information """ if self.file_name is None: return [] with fits.open(self.file_name) as hdul: table = Table(hdul[get_table_hdu_number(hdul)].data) star_data = table.as_array() stars = [] for i in range(len(table)): stars.append( IStar( star_name=get_star_attribute(star_data[i], 0), x=get_star_attribute(star_data[i], 1), y=get_star_attribute(star_data[i], 2), magnitude=get_star_attribute(star_data[i], 5), counts=get_star_attribute(star_data[i], 7), )) return stars
def test_write_create_dataset_kwargs(tmpdir): test_file = str(tmpdir.join('test.hdf5')) the_path = 'the_table' import h5py with h5py.File(test_file, 'w') as output_file: t1 = Table() t1.add_column(Column(name='a', data=[1, 2, 3])) t1.write(output_file, path=the_path, maxshape=(None, )) # A roundabout way of checking this, but the table created above should be # resizable if the kwarg was passed through successfully t2 = Table() t2.add_column(Column(name='a', data=[4, 5])) with h5py.File(test_file, 'a') as output_file: output_file[the_path].resize((len(t1) + len(t2), )) output_file[the_path][len(t1):] = t2.as_array() t3 = Table.read(test_file, path='the_table') assert np.all(t3['a'] == [1, 2, 3, 4, 5])
def test_data_in_other_hdu(self, tmpdir): tbl = Table(np.ones(10).reshape(5, 2)) data = 100 * np.ones(self.shape) hdul = fits.HDUList(hdus=[ fits.PrimaryHDU(), fits.TableHDU(tbl.as_array()), fits.ImageHDU(data) ]) fname = tmpdir.join('test_table.fits').strpath hdul.writeto(fname) logs = [] lh = log_to_list(logger, logs, full_record=True) f = _extract_fits(fname) assert_equal(f['data'], 100 * np.ones(self.shape)) assert_equal(f['unit'], None) # ensure log emitting logs = [i for i in logs if i.message == 'First hdu with image data: 2'] assert_equal(len(logs), 1) assert_equal(logs[0].levelname, 'INFO') logger.removeHandler(lh)
def to_fits_file(self, path): """ Write the visibilities to a fits file. Parameters ---------- path : 'basestr' Path to fits file Returns ------- """ primary_hdu = fits.PrimaryHDU() primary_hdu.header['source'] = 'xrayvision' vis_table = Table([ self.uv.value.T, self.vis, np.repeat([self.xyoffset.value], self.vis.shape, axis=0), np.repeat([self.pixel_size.value], self.vis.shape, axis=0) ], names=('uv', 'vis', 'xyoffset', 'pixel_size')) vis_hdu = fits.BinTableHDU.from_columns( fits.ColDefs(vis_table.as_array())) if self.uv.unit.bases == self.xyoffset.unit.bases == self.pixel_size.unit.bases: vis_hdu.header.set('unit', str(self.uv.unit.bases[0])) else: raise ValueError( f'Units must have the same base unit uv: {self.uv.unit}, xyoffset: ' f'{self.xyoffset.unit}, pixel_size: {self.pixel_size.unit}') hdul = fits.HDUList([primary_hdu, vis_hdu]) try: hdul.writeto(path) except Exception as e: raise e
def convert_to_hdulist(ds): """ transform TheCannon dataset into fits HDU list """ print('@Bo Zhang: ---------------------------------------------------') print('@Bo Zhang: transforming TheCannon data cubes into HDU list ...') print('@Bo Zhang: ---------------------------------------------------') # initialize data list data_list = [ds.wl, ds.ranges, ds.contmask*1, ds.tr_ID, ds.tr_SNR, ds.tr_flux, ds.tr_ivar, ds.tr_label, ds.test_ID, ds.test_SNR, ds.test_flux, ds.test_ivar] name_list = ['wl', 'ranges', 'contmask', 'tr_ID', 'tr_SNR', 'tr_flux', 'tr_ivar', 'tr_label', 'test_ID', 'test_SNR', 'test_flux', 'test_ivar'] hdu_format_list_rw = ['image', 'image', 'image', 'table', 'image', 'image', 'image', 'image', 'table', 'image', 'image', 'image'] # construct Primary header header = fits.Header() header['author'] = 'Bo Zhang (@NAOC)' header['version'] = 'v1.0' header['last_modified'] = 'Sat 11 Jun 2016' header['data'] = 'TheCannon dataset object' # initialize HDU list print('@Bo Zhang: initializing the HDU list ...') hl = [fits.hdu.PrimaryHDU(header=header)] # construct HDU list assert len(data_list) == len(name_list) n_hdus = len(data_list) for i in xrange(n_hdus): print('@Bo Zhang: transforming HDU [%d/%d]: %s ...' % (i+1, n_hdus, name_list[i])) if hdu_format_list_rw[i] == 'table': data = Table(data=data_list[i].reshape((-1, 1)), names=[name_list[i]]) hl.append(fits.BinTableHDU(data.as_array())) else: hl.append(fits.ImageHDU(data=np.array(data_list[i]), name=name_list[i])) print('@Bo Zhang: ---------------------------------------------------') return fits.HDUList(hl)
def inCCARegionTable(self,cfg_par): lineInfo = tP.openLineList(cfg_par) lineThresh = float(lineInfo['SNThresh'][0]) f = fits.open(cfg_par['general']['dataCubeName']) momHead = f[0].header f.close() hdul = fits.open(cfg_par['general']['outTableName']) anc = hdul['ancels'+cfg_par['gFit']['modName']].data bins = hdul['BININFO'].data if not cfg_par['ancillary']['coldGas']['enable']== True: linesG1 = hdul['LineRes_G1'].data if cfg_par['gFit']['modName'] == 'BF': cfg_par['gFit']['modName'] = 'g2' residuals = hdul['Residuals_'+cfg_par['gFit']['modName']].data cfg_par['gFit']['modName'] = 'BF' x=anc['logCentroid_'+cfg_par['ancillary']['coldGas']['Name']] y=anc['logDispIntr_'+cfg_par['ancillary']['coldGas']['Name']] else: x=anc['logCentroid_'+cfg_par['ancillary']['coldGas']['Name']] y=anc['logSigma_'+cfg_par['ancillary']['coldGas']['Name']] if cfg_par['ancillary']['theoreticalCCA'] == 'Ensemble': Mean_sigmav = 2.13 RMS_sigmav = 0.13 Mean_vshift = 1.59 RMS_vshift = 0.37 theta = -179.88 #covariance angle (contours inclined) ellColor='purple' if cfg_par['ancillary']['theoreticalCCA'] == 'Pencil': Mean_sigmav = 1.65 RMS_sigmav = 0.41 Mean_vshift = 2.0 RMS_vshift = 0.47 theta = 165.16 #covariance angle (contours inclined) ellColor='darkseagreen' rmsToFWHM = 2.*np.sqrt(2.*np.log(2)) ellWidth = rmsToFWHM*RMS_vshift ellHeight = rmsToFWHM*RMS_sigmav cos_angle = np.cos(np.radians(180.-theta)) sin_angle = np.sin(np.radians(180.-theta)) xc = x - Mean_vshift yc = y - Mean_sigmav xct = xc * cos_angle - yc * sin_angle yct = xc * sin_angle + yc * cos_angle rad_cc = (xct**2/(ellWidth/2.)**2) + (yct**2/(ellHeight/2.)**2) CCAvec = np.empty(len(anc['BIN_ID']))*np.nan CCAMap = np.zeros([momHead['NAXIS2'],momHead['NAXIS1']])*np.nan if cfg_par['ancillary']['plotRotation'] == True: if cfg_par['ancillary']['plotRotation'] == True: rotModCol=anc['RotMod'] RotMap = np.zeros([momHead['NAXIS2'],momHead['NAXIS1']])*np.nan RotMapCCA = np.zeros([momHead['NAXIS2'],momHead['NAXIS1']])*np.nan for i in range(0,len(rad_cc)): match_bin = np.where(bins['BIN_ID']==anc['BIN_ID'][i])[0] for index in match_bin: if not cfg_par['ancillary']['coldGas']['enable']== True: thresHold = residuals['SN_NII6583'][index] sigmaThresh = linesG1['g1_SigIntr_NII6583'][index] #if thresHold >= lineThresh and sigmaThresh < cfg_par['moments']['sigmaThresh']: if thresHold >= lineThresh: if rad_cc[i] <= 1.: # point in ellipse CCAvec[i] = 1 else: # point not in ellipse CCAvec[i] = 0 CCAMap[int(bins['PixY'][index]),int(bins['PixX'][index])] = CCAvec[i] RotMap[int(bins['PixY'][index]),int(bins['PixX'][index])] = rotModCol[index] else: if not np.isnan(x[i]) and not np.isnan(y[i]): if rad_cc[i] <= 1.: # point in ellipse CCAvec[i] = 1 else: # point not in ellipse CCAvec[i] = 0 CCAMap[int(bins['PixY'][index]),int(bins['PixX'][index])] = CCAvec[i] RotMap[int(bins['PixY'][index]),int(bins['PixX'][index])] = rotModCol[index] if 'CUNIT3' in momHead: del momHead['CUNIT3'] if 'CTYPE3' in momHead: del momHead['CTYPE3'] if 'CDELT3' in momHead: del momHead['CDELT3'] if 'CRVAL3' in momHead: del momHead['CRVAL3'] if 'CRPIX3' in momHead: del momHead['CRPIX3'] if 'NAXIS3' in momHead: del momHead['NAXIS3'] if 'CRDER3' in momHead: del momHead['CRDER3'] Head = momHead.copy() Head['WCSAXES'] = 2 Head['SPECSYS'] = 'topocent' Head['BUNIT'] = 'Jy' fits.writeto(cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'.fits',CCAMap,Head,overwrite=True) if cfg_par['ancillary']['plotRotation'] == True: fits.writeto(cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'rot.fits',RotMap,Head,overwrite=True) fits.writeto(cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'rotCCA.fits',RotMapCCA,Head,overwrite=True) mPl.momAncPlot(cfg_par, cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'.fits', cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],'ancillary') mPl.momAncPlot(cfg_par, cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'rot.fits', cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],'ancillary') mPl.momAncPlot(cfg_par, cfg_par['general']['momModDir']+'ccaMap-'+cfg_par['ancillary']['coldGas']['Name']+'rotCCA.fits', cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],cfg_par['ancillary']['coldGas']['Name'],'ancillary') t=Table(anc) if 'CCAIN' not in anc.dtype.names: t.add_column(Column(CCAvec,name='CCAIN')) else: t.replace_column('CCAIN',Column(CCAvec,name='CCAIN')) hdul['Ancels'+cfg_par['gFit']['modName']] = fits.BinTableHDU(t.as_array(),name='Ancels'+cfg_par['gFit']['modName']) hdul.writeto(cfg_par['general']['outTableName'],overwrite=True) return
def inject_psf(image, mag, coord, psf=None, seed=None): """Realize the DES_PSFEx PSF model `psf` at location `coord` on ZTF science image `image` (image path) with magnitude `mag` in the AB system, fluctuated by Poisson noise. """ # initialize the random number generator import galsim rng = galsim.BaseDeviate(seed) # handle both scalar and vector inputs mag = np.atleast_1d(mag) if coord.isscalar: coord = coord.reshape([1]) with fits.open(image, mode='update') as hdul: # read in the WCS header = hdul[0].header wcs = WCS(header=header) # measure the PSF using PSFEx if not already specified if psf is None: psf = measure_psf(image) # load the image into galsim gimage = galsim.fits.read(hdu_list=hdul) # convert the world coordinates to pixel coordinates ipos = wcs.all_world2pix([[pos.ra.deg, pos.dec.deg] for pos in coord], 1) for m, pos in zip(mag, ipos): # calculate the measured flux of the object flux = 10**(-0.4 * (m - header['MAGZP'])) image_pos = galsim.PositionD(*pos) # store the center of the nearest integer pixel iimage_pos = galsim.PositionI(*tuple(map(round, pos))) ix, iy = iimage_pos.x, iimage_pos.y # calculate the offset between the stamp center # and the profile center offset = image_pos - iimage_pos # create an output stamp onto which to draw the fluctuated # psf bounds = galsim.BoundsI(ix - NPIX, ix + NPIX, iy - NPIX, iy + NPIX) # check that there is at least some overlap between the bounds = bounds & gimage.bounds if not bounds.isDefined(): raise RuntimeError('No overlap between PSF stamp and image. ' 'Is the object coordinate contained by the ' 'image?') # get the noise noise = galsim.PoissonNoise(rng) # realize the psf at the coordinates realization = psf.getPSF(image_pos).withFlux(flux) # get the local wcs lwcs = psf.getLocalWCS(image_pos) # draw the image imout = realization.drawImage(wcs=lwcs, offset=offset, nx=NPIX * 2 + 1, ny=NPIX * 2 + 1) # add the noise imout.addNoise(noise) # shift the image to the right spot imout.setCenter(iimage_pos) # add the photons gimage[bounds] = gimage[bounds] + imout[bounds] # save it as a new hdu galsim.fits.write(gimage, hdu_list=hdul) # propagate original WCS to output extension wcskeys = wcs.to_header(relax=True) hdul[-1].header.update(wcskeys) # add a record of the fakes as a bintable record = { 'fake_mag': mag, 'fake_ra': coord.ra.deg, 'fake_dec': coord.dec.deg, 'fake_x': ipos[:, 0], 'fake_y': ipos[:, 1] } # save the bintable as an extension table = Table(record) nhdu = fits.BinTableHDU(table.as_array()) hdul.append(nhdu)
class BasicCatalog(object): def __init__(self,src_names,lon,lat,significance,unit='deg',frame='FK5',_selected=None,_table=None): self.selected = np.ones(len(src_names), dtype=np.bool) if _selected is not None: self.selected = False self.selected[_selected] = True self._sc = SkyCoord(lon,lat,frame=frame, unit=unit) self.lat_name, self.lon_name=self.get_coord_names(self.sc) meta={'FRAME':frame} meta['COORD_UNIT']=unit meta['LON_NAME']=self.lon_name meta['LAT_NAME']=self.lat_name if _table is None: self._table = Table([np.arange(len(src_names)),src_names, significance, lon, lat], names=['meta_ID','src_names', 'significance', self.lon_name, self.lat_name],meta=meta,masked=True) else: self._table=Table(_table.as_array(),names=_table.colnames,meta=meta,masked=True) def select_IDs(self,ids): self.unselect_all() self.selected[ids]=True def select_all(self): self.selected[::]=True def unselect_all(self): self.selected[::]=False def get_coord_names(self,sc): inv_map = {v: k for k, v in sc.representation_component_names.items()} _lat_name = inv_map['lat'] _lon_name = inv_map['lon'] return _lat_name,_lon_name @property def table(self): return self._table[self.selected] @property def sc(self): return self._sc[self.selected] @property def length(self): return self._table.as_array().shape[0] @property def ra(self): return self.sc.fk5.ra @property def dec(self): return self.sc.fk5.dec @property def l(self): return self.sc.galactic.l @property def b(self): return self.sc.galactic.b @property def name(self): return self.table['src_names'] @property def significance(self): return self.table['significance'] @property def lat(self): return self.table[ self.lat_name] @property def lon(self): return self.table[ self.lon_name] def add_column(self,data=None,name=None,dtype=None): if data is None: data=np.zeros(self.table.as_array().shape[0]) self._table.add_column(Column(data=data,name=name,dtype=dtype)) def get_dictionary(self ): column_lists=[self.table[name].tolist() for name in self.table.colnames] for ID,_col in enumerate(column_lists): column_lists[ID] = [x if str(x)!='nan' else None for x in _col] return dict(cat_frame=self.table.meta['FRAME'], cat_coord_units=self.table.meta['COORD_UNIT'], cat_column_list=column_lists, cat_column_names=self.table.colnames, cat_column_descr=self.table.dtype.descr, cat_lat_name=self.lat_name, cat_lon_name=self.lon_name) def write_ds9_region(self,name,overwrite=True): ra=self.sc.fk5.ra dec=self.sc.fk5.dec src_names=self.name with open(name,'w') as f: for r,d,src_name in zip(ra,dec,src_names): s=u'''fk5; point %f %f #point = x text = {%s} \n'''%(r.deg,d.deg,src_name) f.write(s) # #user_catalog = BasicCatalog(src_names, lon, lat, significance, _table=t, unit=unit, frame=frame) def decode(self,enc_table): pass def encode(self,): _table=numpy_encode(self.table.as_array()) _meta=dumps(self.table.meta) return dumps(_table,_meta) def write(self,name,format='fits',overwrite=True): self._table.write(name,format=format,overwrite=overwrite) @classmethod def from_ecsv_file(cls, file_name): return cls.from_table(Table.read(file_name, format='ascii.ecsv')) @classmethod def from_fits_file(cls,file_name): return cls.from_table(Table.read(file_name,format='fits')) @classmethod def from_file(cls,file_name): format_list=['ascii.ecsv','fits'] cat=None for f in format_list: try: cat= cls.from_table(Table.read(file_name,format=f)) except: pass if cat is None: raise RuntimeError('file format for catalog not valid') return cat @classmethod def from_table(cls,table): try: src_names=table['src_names'] significance=table['significance'] frame = table.meta['FRAME'] lon=table[table.meta['LON_NAME']] lat =table[table.meta['LAT_NAME']] unit = table.meta['COORD_UNIT'] cat= cls(src_names,lon,lat,significance,_table=table,unit=unit,frame=frame) except: raise RuntimeError('Table in fits file is not valid to build Catalog') return cat
def main(argv=None): """ Main Function """ # Call initial parser from init_utils parser = ap.ArgumentParser(description="""Create HDF5 Survey file.""", add_help=True) parser.add_argument( "-r", "--rootdir", help="""Root Directory for Reductions""", type=str, default="/work/03946/hetdex/maverick/red1/reductions/", ) parser.add_argument( "-sdir", "--shotdir", help="""Directory for shot H5 files to ingest""", type=str, default="/scratch/03946/hetdex/hdr3/reduction/data", ) parser.add_argument( "-sl", "--shotlist", help="""Text file of DATE OBS list""", type=str, default="dex.hdr2.shotlist", ) parser.add_argument( "-ad", "--astrometry_dir", help="""Directory for Shifts""", type=str, default="/data/00115/gebhardt/vdrp/shifts/", ) parser.add_argument( "-of", "--outfilename", type=str, help="""Relative or absolute path for output HDF5 file.""", default=None, ) parser.add_argument( "-flim", "--flim", help="""Path to flim look up table""", type=str, default="/data/05350/ecooper/hdr2.1/survey/average_one_sigma.txt", ) parser.add_argument("-survey", "--survey", type=str, default="hdr2.1") args = parser.parse_args(argv) print(args) args.log = setup_logging() fileh = tb.open_file(args.outfilename, mode="w", title=args.survey.upper() + "Survey file ") shotlist = Table.read(args.shotlist, format="ascii.no_header", names=["date", "obs"]) survey = Table() for shotrow in shotlist: datevshot = str(shotrow["date"]) + "v" + str(shotrow["obs"]).zfill(3) if True: args.log.info('Ingesting ' + datevshot) file_obs = tb.open_file(op.join(args.shotdir, datevshot + ".h5"), "r") shottable = Table(file_obs.root.Shot.read()) # updating field in survey file shottable['field'] = define_field(str(shottable['objid'][0])) survey = vstack([survey, shottable]) file_obs.close() else: #except: args.log.error("Could not ingest %s" % datevshot) tableMain = fileh.create_table(fileh.root, "Survey", obj=survey.as_array()) tableMain.flush() fileh.close()
def gather_nightwatch_qa(night, verbose=False, overwrite=False): """Read and stack all the nightwatch QA files for a given night. """ import json import astropy.table from astropy.table import Table qadir = os.path.join(outdir, 'nightwatch') if not os.path.isdir(qadir): os.makedirs(qadir, exist_ok=True) stackwatchfile = os.path.join(qadir, 'qa-nightwatch-{}.fits'.format(str(night))) fiberassignmapfile = os.path.join( qadir, 'fiberassignmap-{}.fits'.format(str(night))) if os.path.isfile(stackwatchfile) and not overwrite: print('Reading {}'.format(stackwatchfile)) data = Table(fitsio.read(stackwatchfile, 'PER_CAMFIBER')) fiberassignmap = Table(fitsio.read(fiberassignmapfile)) else: #print('Reading the focal plane model.') #fp = desimodel.io.load_focalplane()[0] #fp = fp['PETAL', 'FIBER', 'OFFSET_X', 'OFFSET_Y'] nightdir = os.path.join(nightwatchdir, str(night)) allexpiddir = glob(os.path.join(nightdir, '????????')) data = [] fiberassignmap = Table(names=('NIGHT', 'EXPID', 'TILEID', 'FIBERASSIGNFILE'), dtype=('U8', 'U8', 'i4', 'U32')) for expiddir in allexpiddir: expid = os.path.basename(expiddir) qafile = os.path.join(expiddir, 'qa-{}.fits'.format(expid)) qaFITS = fitsio.FITS(qafile) if 'PER_CAMFIBER' in qaFITS: if verbose: print('Reading {}'.format(qafile)) qa = Table(qaFITS['PER_CAMFIBER'].read()) # Hack! Figure out the mapping between EXPID and FIBERMAPusing the request-EXPID.json file. requestfile = os.path.join(rawdata_dir, str(night), expid, 'request-{}.json'.format(expid)) if not os.path.isfile(requestfile): print('Missing {}'.format(requestfile)) continue with open(requestfile) as ff: req = json.load(ff) if 'PASSTHRU' in req.keys(): if type(req['PASSTHRU']) is dict: tileid = req['PASSTHRU']['TILEID'] #tileid = int(req['PASSTHRU'].split(':')[3].split(',')[0]) else: indx = req['PASSTHRU'].index('TILEID') tileid = req['PASSTHRU'][indx:] tileid = int(tileid[tileid.index(':') + 1:tileid.index(',')]) # This should use the svn checkout! tilefile = glob( os.path.join(rawdata_dir, str(night), '????????', 'fiberassign-{:06d}.fits'.format(tileid))) #if len(tilefile) == 0: # print('No fibermap file found for EXPID={}'.format(expid)) #if len(tilefile) > 0: # print('Multiple fibermap files found for EXPID={}!'.format(expid)) if len(tilefile) > 0: tsplit = tilefile[0].split('/') fiberassignmap.add_row( (str(night), str(expid), tileid, os.path.join(tsplit[-2], tsplit[-1]))) #fiberassignmap[str(expid)] = [tileid] data.append(qa) #else: # print(' No tilefile found') #else: # print(' No tilefile found') if len(data) == 0: print('No fiberassign files found for night {}'.format(night)) return None, None data = astropy.table.vstack(data) # Need to update the data model to 'f4'. print('Updating the data model.') for col in data.colnames: if data[col].dtype == '>f8': data[col] = data[col].astype('f4') print('Writing {}'.format(stackwatchfile)) fitsio.write(stackwatchfile, data.as_array(), clobber=True, extname='PER_CAMFIBER') print('Writing {}'.format(fiberassignmapfile)) # ValueError: unsupported type 'U42' #fitsio.write(fiberassignmapfile, fiberassignmap.as_array(), clobber=True) fiberassignmap.write(fiberassignmapfile, overwrite=True) return data, fiberassignmap
import numpy as np e_bins = 20 o_bins = 5 e_axis = np.logspace(-1,2,e_bins+1) o_axis = np.linspace(0,3,o_bins+1) rad_max = (np.ones([e_bins, o_bins]).T * np.linspace(0.1, 0.01, e_bins)).T table = Table([[e_axis[:-1]],[e_axis[1:]], [o_axis[:-1]],[o_axis[1:]], [rad_max]], names=('ENERG_LO', 'ENERG_HI', 'THETA_LO', 'THETA_HI', 'RAD_MAX')) data = table.as_array() header = fits.Header() header['HDUDOC'] = 'https://github.com/open-gamma-ray-astro/gamma-astro-data-formats', '' header['HDUVERS'] = '0.2', '' header['HDUCLASS'] = 'GADF', '' header['HDUCLAS1'] = 'RESPONSE', '' header['HDUCLAS2'] = 'RAD_MAX', '' header['HDUCLAS3'] = 'POINT-LIKE', '' header['HDUCLAS4'] = 'RAD_MAX_2D', '' tbhdu = fits.BinTableHDU(data, header, name='RAD_MAX') prihdu = fits.PrimaryHDU() thdulist = fits.HDUList([prihdu, tbhdu])
class TestUserSuppliedHaloCatalog(TestCase): """ Class providing tests of the `~halotools.sim_manager.UserSuppliedHaloCatalog`. """ def setUp(self): """ Pre-load various arrays into memory for use by all tests. """ self.Nhalos = 1e2 self.Lbox = 100 self.redshift = 0.0 self.halo_x = np.linspace(0, self.Lbox, self.Nhalos) self.halo_y = np.linspace(0, self.Lbox, self.Nhalos) self.halo_z = np.linspace(0, self.Lbox, self.Nhalos) self.halo_mass = np.logspace(10, 15, self.Nhalos) self.halo_id = np.arange(0, self.Nhalos, dtype = np.int) self.good_halocat_args = ( {'halo_x': self.halo_x, 'halo_y': self.halo_y, 'halo_z': self.halo_z, 'halo_id': self.halo_id, 'halo_mass': self.halo_mass} ) self.toy_list = [elt for elt in self.halo_x] self.num_ptcl = 1e4 self.good_ptcl_table = Table( {'x': np.zeros(self.num_ptcl), 'y': np.zeros(self.num_ptcl), 'z': np.zeros(self.num_ptcl)} ) self.dummy_cache_baseloc = helper_functions.dummy_cache_baseloc try: shutil.rmtree(self.dummy_cache_baseloc) except: pass os.makedirs(self.dummy_cache_baseloc) def test_particle_mass_requirement(self): with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog(Lbox = 200, **self.good_halocat_args) def test_lbox_requirement(self): with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog(particle_mass = 200, **self.good_halocat_args) def test_halos_contained_inside_lbox(self): with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog(Lbox = 20, particle_mass = 100, **self.good_halocat_args) def test_redshift_is_float(self): with pytest.raises(HalotoolsError) as err: halocat = UserSuppliedHaloCatalog( Lbox = 200, particle_mass = 100, redshift = '1.0', **self.good_halocat_args) substr = "The ``redshift`` metadata must be a float." assert substr in err.value.message def test_successful_load(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) assert hasattr(halocat, 'Lbox') assert halocat.Lbox == 200 assert hasattr(halocat, 'particle_mass') assert halocat.particle_mass == 100 def test_additional_metadata(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, arnold_schwarzenegger = 'Stick around!', **self.good_halocat_args) assert hasattr(halocat, 'arnold_schwarzenegger') assert halocat.arnold_schwarzenegger == 'Stick around!' def test_all_halo_columns_have_length_nhalos(self): # All halo catalog columns must have length-Nhalos bad_halocat_args = deepcopy(self.good_halocat_args) with pytest.raises(HalotoolsError): bad_halocat_args['halo_x'][0] = -1 halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) def test_positions_contained_inside_lbox_alt_test(self): # positions must be < Lbox bad_halocat_args = deepcopy(self.good_halocat_args) with pytest.raises(HalotoolsError): bad_halocat_args['halo_x'][0] = 10000 halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) def test_has_halo_x_column(self): # must have halo_x column bad_halocat_args = deepcopy(self.good_halocat_args) with pytest.raises(HalotoolsError): del bad_halocat_args['halo_x'] halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) def test_has_halo_id_column(self): # Must have halo_id column bad_halocat_args = deepcopy(self.good_halocat_args) with pytest.raises(HalotoolsError): del bad_halocat_args['halo_id'] halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) def test_has_halo_mass_column(self): # Must have some column storing a mass-like variable bad_halocat_args = deepcopy(self.good_halocat_args) with pytest.raises(HalotoolsError): del bad_halocat_args['halo_mass'] halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) def test_halo_prefix_warning(self): # Must raise warning if a length-Nhalos array is passed with # a keyword argument that does not begin with 'halo_' bad_halocat_args = deepcopy(self.good_halocat_args) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") bad_halocat_args['s'] = np.ones(self.Nhalos) halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **bad_halocat_args) assert 'interpreted as metadata' in str(w[-1].message) def test_ptcl_table(self): """ Method performs various existence and consistency tests on the input ptcl_table. * Enforce that instances do *not* have ``ptcl_table`` attributes if none is passed. * Enforce that instances *do* have ``ptcl_table`` attributes if a legitimate one is passed. * Enforce that ptcl_table have ``x``, ``y`` and ``z`` columns. * Enforce that ptcl_table input is an Astropy `~astropy.table.Table` object, not a Numpy recarray """ def test_ptcl_table_dne(self): # Must not have a ptcl_table attribute when none is passed halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) assert not hasattr(halocat, 'ptcl_table') def test_ptcl_table_exists_when_given_goodargs(self): # Must have ptcl_table attribute when argument is legitimate halocat = UserSuppliedHaloCatalog( Lbox = 200, particle_mass = 100, redshift = self.redshift, ptcl_table = self.good_ptcl_table, **self.good_halocat_args) assert hasattr(halocat, 'ptcl_table') def test_min_numptcl_requirement(self): # Must have at least 1e4 particles num_ptcl2 = 1e3 ptcl_table2 = Table( {'x': np.zeros(num_ptcl2), 'y': np.zeros(num_ptcl2), 'z': np.zeros(num_ptcl2)} ) with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog( Lbox = 200, particle_mass = 100, redshift = self.redshift, ptcl_table = ptcl_table2, **self.good_halocat_args) def test_ptcls_have_zposition(self): # Must have a 'z' column num_ptcl2 = 1e4 ptcl_table2 = Table( {'x': np.zeros(num_ptcl2), 'y': np.zeros(num_ptcl2)} ) with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog( Lbox = 200, particle_mass = 100, redshift = self.redshift, ptcl_table = ptcl_table2, **self.good_halocat_args) def test_ptcls_are_astropy_table(self): # Data structure must be an astropy table, not an ndarray ptcl_table2 = self.good_ptcl_table.as_array() with pytest.raises(HalotoolsError): halocat = UserSuppliedHaloCatalog( Lbox = 200, particle_mass = 100, redshift = self.redshift, ptcl_table = ptcl_table2, **self.good_halocat_args) @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache1(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc' fname = os.path.join(self.dummy_cache_baseloc, basename) os.system('touch ' + fname) assert os.path.isfile(fname) dummy_string = ' ' with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( fname, dummy_string, dummy_string, dummy_string, dummy_string) substr = "Either choose a different fname or set ``overwrite`` to True" assert substr in err.value.message with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( fname, dummy_string, dummy_string, dummy_string, dummy_string, overwrite = True) assert substr not in err.value.message @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache2(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc' dummy_string = ' ' with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( basename, dummy_string, dummy_string, dummy_string, dummy_string) substr = "The directory you are trying to store the file does not exist." assert substr in err.value.message @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache3(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc' fname = os.path.join(self.dummy_cache_baseloc, basename) os.system('touch ' + fname) assert os.path.isfile(fname) dummy_string = ' ' with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( fname, dummy_string, dummy_string, dummy_string, dummy_string, overwrite = True) substr = "The fname must end with an ``.hdf5`` extension." assert substr in err.value.message @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache4(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc.hdf5' fname = os.path.join(self.dummy_cache_baseloc, basename) os.system('touch ' + fname) assert os.path.isfile(fname) dummy_string = ' ' class Dummy(object): pass def __str__(self): raise TypeError not_representable_as_string = Dummy() with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( fname, not_representable_as_string, dummy_string, dummy_string, dummy_string, overwrite = True) substr = "must all be strings." assert substr in err.value.message @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache5(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc.hdf5' fname = os.path.join(self.dummy_cache_baseloc, basename) os.system('touch ' + fname) assert os.path.isfile(fname) dummy_string = ' ' class Dummy(object): pass def __str__(self): raise TypeError not_representable_as_string = Dummy() with pytest.raises(HalotoolsError) as err: halocat.add_halocat_to_cache( fname, dummy_string, dummy_string, dummy_string, dummy_string, overwrite = True, some_more_metadata = not_representable_as_string) substr = "keyword is not representable as a string." assert substr in err.value.message @pytest.mark.skipif('not HAS_H5PY') def test_add_halocat_to_cache6(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) basename = 'abc.hdf5' fname = os.path.join(self.dummy_cache_baseloc, basename) simname = 'dummy_simname' halo_finder = 'dummy_halo_finder' version_name = 'dummy_version_name' processing_notes = 'dummy processing notes' halocat.add_halocat_to_cache( fname, simname, halo_finder, version_name, processing_notes, overwrite = True, some_additional_metadata = processing_notes) cache = HaloTableCache() assert halocat.log_entry in cache.log cache.remove_entry_from_cache_log( halocat.log_entry.simname, halocat.log_entry.halo_finder, halocat.log_entry.version_name, halocat.log_entry.redshift, halocat.log_entry.fname, raise_non_existence_exception = True, update_ascii = True, delete_corresponding_halo_catalog = True) @pytest.mark.skipif('not HAS_H5PY') @pytest.mark.xfail def test_add_ptcl_table_to_cache(self): halocat = UserSuppliedHaloCatalog(Lbox = 200, particle_mass = 100, redshift = self.redshift, **self.good_halocat_args) halocat.add_ptcl_table_to_cache() def tearDown(self): try: shutil.rmtree(self.dummy_cache_baseloc) except: pass
'length': length, 'phi': phi, 'psi': psi, 'r': r, 'cen_x': cen_x, 'cen_y': cen_y, 'size': size, 'mcAlttel': mcAlttel, 'mcAztel': mcAztel } ntuple = Table(output) #If destination fitsfile doesn't exist, will create a new one with proper headers if os.path.isfile(outfile) == False: #Convert Tables of data into HDUBinTables to write them into fits files pardata = ntuple.as_array() parheader = fits.Header() parheader.update(ntuple.meta) if storeimg == True: pixels = fits.ImageHDU(fitsdata) #Image with pixel content #Write the data in an HDUList for storing in a fitsfile hdr = fits.Header( ) #Example header, we can add more things to this header hdr['TEL'] = 'LST1' primary_hdu = fits.PrimaryHDU(header=hdr) hdul = fits.HDUList([primary_hdu]) hdul.append(fits.BinTableHDU(data=pardata, header=parheader)) if storeimg == True: hdul.append(pixels)
def test_write_bintable(self): """Test writing binary tables to FITS. """ from ..io.util import write_bintable, fitsheader # # Input: Table # hdr = fitsheader(dict(A=1, B=2)) hdr['C'] = ('BLAT', 'FOO') data = Table() data['X'] = [1, 2, 3] data['Y'] = [3, 4, 5] write_bintable(self.testfile, data, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: ndarray # hdr = dict(A=1, B=2) data = data.as_array() write_bintable(self.testfile, data, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: dictionary # hdr = dict(A=1, B=2) d = dict(X=np.array([1, 2, 3]), Y=np.array([3, 4, 5])) write_bintable(self.testfile, d, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: Table with column comments. # hdr = fitsheader(dict(A=1, B=2)) hdr['C'] = ('BLAT', 'FOO') data = Table() data['X'] = [1, 2, 3] data['Y'] = [3, 4, 5] write_bintable(self.testfile, data, header=hdr, comments={'X': 'This is X', 'Y': 'This is Y'}, units={'X': 'mm', 'Y': 'mm'}) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) self.assertEqual(newhdr['TTYPE1'], 'X') self.assertEqual(newhdr.comments['TTYPE1'], 'This is X') self.assertEqual(newhdr['TTYPE2'], 'Y') self.assertEqual(newhdr.comments['TTYPE2'], 'This is Y') self.assertEqual(newhdr['TUNIT1'], 'mm') self.assertEqual(newhdr.comments['TUNIT1'], 'X units') self.assertEqual(newhdr['TUNIT2'], 'mm') self.assertEqual(newhdr.comments['TUNIT2'], 'Y units') # # Input: Table with no EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr) # # Input: Table with EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR') # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True, extname='FOOBAR') self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) # # Input: Table with existing EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR') # # Input: Table with EXTNAME, existing file, overwrite # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR', clobber=True)
def importDATA(password): #import input data from MIGO table Special_id1, Obs_date, Vertex1, Vertex2, Vertex3, Vertex4, Status, Updated_Date = Open_InputDatabase.getinputs( password) #loop over each special ID for x in range(0, 2): #len(Special_id1)): #import url and make a table url, z, TYPE = url_for_ISPY_SkyCoord.ISPY_ephemeris_inSkyCoord( 'JWST', Obs_date[x], Special_id1[x], Vertex1[x], Vertex2[x], Vertex3[x], Vertex4[x]) #get table name and create a table for this Specific ID table_name = Setup_table_inMYSQL_tofillin_data.makeMySQLtable( Special_id1[x], password) Special_id = table_name #check if there are any data on the website if z == None: k = 'No results' else: #get the table and the tablelines distant, tablelines = Create_ISPYtable.make_table( z, 'SPK-ID', 'EXPLANATION', 1, 1) #make the table into lines to put in the database f = Table.as_array(distant) #make a string of JPL_SPKID values JPL_SPKID = [] for x in range(0, len(f)): JPL_SPKID.append(f[x][0]) # Open database connection db = pymysql.connect("localhost", "root", password) # prepare a cursor object using cursor() method and go to the right database cursor = db.cursor() cursor.execute("USE ISPY") #table columns tab_col = [ 'JPL_SPKID', 'IAU_Number', 'Name_designation', 'RA', 'DEC1', 'Amag', 'dRAcosD', 'dDEC_by_dt', 'CntDst', 'PsAng', 'Data_Arc', 'Nobs', 'Error', 'Ellipse', 'Theta', 'Last_updated', table_name, 'Special_id', Special_id ] #check if there are any data in the table already sql3 = "SELECT {0} from {1}".format(tab_col[0], tab_col[16]) try: # Execute the SQL command cursor.execute(sql3) #Fetch all the rows in a list of lists. These are the data which are already in the database result = cursor.fetchall() except: print("Error: unable to fetch data") #select the data from the table which are older than 1 month sql4 = "SELECT {0} from {1} where {2}<NOW()-INTERVAL 1 MONTH".format( tab_col[0], tab_col[16], tab_col[15]) try: # Execute the SQL command cursor.execute(sql4) # Fetch all the rows in a list of lists. these are the data which are older than 1 month result1 = cursor.fetchall() # Now print fetched result except: print('No data to return') #loop over all the rows in the table from ISPY for x in range(0, len(f)): #check if the asteroid already in the database if JPL_SPKID[x] in result: #print ('Already in the database') #check how old are the data of the asteroid in the database (if old update them) if JPL_SPKID[x] in result1: sql1 = "UPDATE {0} SET RA='{1[3]}',DEC1='{1[4]}',Special_id='{2}' where JPL_SPKID={1[0]}".format( tab_col[16], f[x], tab_col[18]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #update Status in the input table to know if that line has been run already sql11 = "UPDATE INPUT_Table SET Status='UPDATED' where Special_id='{0}'".format( tab_col[16]) #print (sql11) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #loop over all the columns, take care for the empty data columns/rows for y in range(0, len(tab_col) - 4): if y == 0 or y == 2 or y == 3 or y == 4: sql1 = 0 elif ma.getmask(f[x][y]) == True: # Prepare SQL query to INSERT a record into the database. sql1 = 0 #if data is missing, nothing is imported into the database (the value becomes None) else: #fill in(update) the data sql1 = "UPDATE [0] SET {1}={2}\ where {3}={4}".format( tab_col[16], tab_col[y], f[x][y], tab_col[0], f[x][0]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #update Status in the input table to know if that line has been run already sql11 = "UPDATE INPUT_Table SET Status='UPDATED' where Special_id='{0}'".format( tab_col[16]) #print (sql11) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #if the data is not in the database, input them else: sql1 = "INSERT INTO {0}(Special_id,JPL_SPKID,Name_designation,\ RA ,\ DEC1 ,Last_Updated)\ VALUES ('{2}',{1[0]},'{1[2]}','{1[3]}','{1[4]}',NOW())".format( tab_col[16], f[x], tab_col[18]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #loop over all the columns for y in range(0, len(tab_col) - 4): if y == 0 or y == 2 or y == 3 or y == 4: sql1 = 0 elif ma.getmask(f[x][y]) == True: # Prepare SQL query to INSERT a record into the database. sql1 = 0 else: sql1 = "UPDATE {0} SET {1}={2}\ where {3}={4}".format(tab_col[16], tab_col[y], f[x][y], tab_col[0], f[x][0]) #rint (x,y,f[x][y]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #update Status in the input table to know if that line has been run already sql11 = "UPDATE INPUT_Table SET Status='UPDATED' where Special_id='{0}'".format( tab_col[16]) #print (sql11) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() #check if the number of rows in f and in MySQL is the same (so if all the data has been imported) sql2 = "SELECT COUNT(*) FROM {0}".format(tab_col[16]) try: # Execute the SQL command cursor.execute(sql2) # Fetch all the rows in a list of lists. result2 = cursor.fetchone() # Now check fetched result if result2[0] == len(f): print( 'For Special ID ', Special_id, ': All the data have been recorded into the database.') else: #print ('For Special ID ',Special_id,': The number of rows in the database is not the same as from ISPY') #check why the sizes of tables do not match sql5 = "SELECT {0} from {1}".format( tab_col[0], tab_col[16]) try: # Execute the SQL command cursor.execute(sql5) #Fetch all the rows in a list of lists. result5 = cursor.fetchall() for x in range(0, len(result5)): #check if the asteroid already in the database if result5[x] not in JPL_SPKID: # Prepare SQL query to DELETE required records sql6 = "DELETE FROM {0} WHERE {1}={2}".format( tab_col[16], tab_col[0], result5[x][0]) #print (sql6) try: # Execute the SQL command cursor.execute(sql6) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() print( 'Not possible to delete the outdated data' ) except: print( "Error: unable to fetch data to find the reason for not matching tables." ) try: # Execute the SQL command cursor.execute(sql2) # Fetch all the rows in a list of lists. result2 = cursor.fetchone() # Now check fetched result if result2[0] == len(f): print( 'For Special ID ', Special_id, ': All the data have been recorded into the database.' ) except: print('For Special ID ', Special_id, ": Error: unable to fetch data") except: print('For Special ID ', Special_id, ": Error: unable to fetch data") # disconnect from server db.close() return Special_id1
synth_sp = [] smooth_sp = [] lambdas = spectra[0][14] colnames_out = colnames[:12] + colnames[15:] width = 50./ (lambdas[1] - lambdas[0]) for row in spectra: # Some spectra are just invalid... if row[12][0] > 0.2: data.append(row[:12] + row[15:]) obs_sp.append(row[12]) smooth_sp.append(row[12] / gaussian_filter1d(row[12], width)) synth_sp.append(row[13]) data_table = Table(rows=data, names=colnames_out, dtype=[type(x) for x in data[0]]) header = fits.Header() header.update(data_table.meta) main = fits.BinTableHDU(data_table.as_array(), header) lambda_column = fits.Column('lambda', 'E', array=lambdas) lambdas = fits.BinTableHDU.from_columns([lambda_column]) header = fits.Header() header['CONTENT'] = 'Observed spectra' obs_sp = fits.ImageHDU(obs_sp, header, do_not_scale_image_data=True) header['CONTENT'] = 'Smoothed spectra' smooth_sp = fits.ImageHDU(smooth_sp, header, do_not_scale_image_data=True) header['CONTENT'] = 'Synthetic spectra' synth_sp = fits.ImageHDU(synth_sp, header, do_not_scale_image_data=True) hdulist = fits.HDUList([hdu, main, obs_sp, smooth_sp, synth_sp, lambdas]) hdulist.writeto('RAVE_ready.fits')
def aper_summary(self, gal_only=False, output=False): """Get all the stellar mass, age, and metallicity profiles. Parameters ---------- gal_only: bool, optional Only provide summary of the whole galaxy. Default: False. output : bool, optional Return the `maper` array when True. Default: False Configuration Parameters ------------------------ Can be found in `self.config`: subpix : int, optional Subpixel sampling factor. Default is 5. """ # Aperture mass profiles self.maper('gal') if not gal_only: self.maper('ins') self.maper('exs') # Aperture age profiles self.aprof('age', 'gal', return_mass=True) if not gal_only: self.aprof('age', 'ins', return_mass=True) self.aprof('age', 'exs', return_mass=True) # Aperture metallicity profiles self.aprof('met', 'gal') if not gal_only: self.aprof('met', 'ins') self.aprof('met', 'exs') # Gather these results into an Astropy Table aper_sum = Table() aper_sum.add_column(Column(data=self.rad_inn, name='rad_inn')) aper_sum.add_column(Column(data=self.rad_out, name='rad_out')) aper_sum.add_column(Column(data=self.rad_mid, name='rad_mid')) aper_sum.add_column(Column(data=self.maper_gal, name='maper_gal')) if not gal_only: aper_sum.add_column(Column(data=self.maper_ins, name='maper_ins')) aper_sum.add_column(Column(data=self.maper_exs, name='maper_exs')) aper_sum.add_column( Column(data=self.age_prof_gal['prof_w'], name='age_gal_w')) aper_sum.add_column( Column(data=self.age_prof_gal['prof'], name='age_gal')) aper_sum.add_column( Column(data=self.age_prof_gal['flag'], name='age_gal_flag')) aper_sum.add_column( Column(data=self.age_prof_gal['mass'], name='mprof_gal')) if not gal_only: aper_sum.add_column( Column(data=self.age_prof_ins['prof_w'], name='age_ins_w')) aper_sum.add_column( Column(data=self.age_prof_ins['prof'], name='age_ins')) aper_sum.add_column( Column(data=self.age_prof_ins['flag'], name='age_ins_flag')) aper_sum.add_column( Column(data=self.age_prof_ins['mass'], name='mprof_ins')) aper_sum.add_column( Column(data=self.age_prof_exs['prof_w'], name='age_exs_w')) aper_sum.add_column( Column(data=self.age_prof_exs['prof'], name='age_exs')) aper_sum.add_column( Column(data=self.age_prof_exs['flag'], name='age_exs_flag')) aper_sum.add_column( Column(data=self.age_prof_exs['mass'], name='mprof_exs')) aper_sum.add_column( Column(data=self.met_prof_gal['prof_w'], name='met_gal_w')) aper_sum.add_column( Column(data=self.met_prof_gal['prof'], name='met_gal')) aper_sum.add_column( Column(data=self.met_prof_gal['flag'], name='met_gal_flag')) if not gal_only: aper_sum.add_column( Column(data=self.met_prof_ins['prof_w'], name='met_ins_w')) aper_sum.add_column( Column(data=self.met_prof_ins['prof'], name='met_ins')) aper_sum.add_column( Column(data=self.met_prof_ins['flag'], name='met_ins_flag')) aper_sum.add_column( Column(data=self.met_prof_exs['prof_w'], name='met_exs_w')) aper_sum.add_column( Column(data=self.met_prof_exs['prof'], name='met_exs')) aper_sum.add_column( Column(data=self.met_prof_exs['flag'], name='met_exs_flag')) setattr(self, 'aper_sum', aper_sum.as_array()) if output: return aper_sum
'phi': phi, 'psi': psi, 'r': r, 'x': x, 'y': y, 'intensity': intensity, 'mcAlttel': mcAlttel, 'mcAztel': mcAztel } ntuple = Table(output) #If destination fitsfile doesn't exist, will create a new one with proper headers if os.path.isfile(outfile) == False: if filetype == 'fits': #Convert Tables of data into HDUBinTables to write them into fits files pardata = ntuple.as_array() parheader = fits.Header() parheader.update(ntuple.meta) if storeimg == True: pixels = fits.ImageHDU(fitsdata) #Image with pixel content #Write the data in an HDUList for storing in a fitsfile hdr = fits.Header( ) #Example header, we can add more things to this header hdr['TEL'] = 'LST1' primary_hdu = fits.PrimaryHDU(header=hdr) hdul = fits.HDUList([primary_hdu]) hdul.append(fits.BinTableHDU(data=pardata, header=parheader)) if storeimg == True: hdul.append(pixels)
def precompute_synthetic_grid(output_dirname, ranges, wavelengths, to_resolution, modeled_layers_pack, atomic_linelist, isotopes, solar_abundances, segments=None, number_of_processes=1, code="spectrum", use_molecules=False, steps=False, tmp_dir=None): """ Pre-compute a synthetic grid with some reference ranges (Teff, log(g) and MH combinations) and all the steps that iSpec will perform in the astrophysical parameter determination process. All the non-convolved spectra will be saved in a subdir and a complete grid file with the reference points already convolved will be saved in a FITS file for fast comparison. The output directory can be used by the routines 'model_spectrum' and 'estimate_initial_ap'. """ code = code.lower() if code not in ['spectrum', 'turbospectrum', 'moog', 'synthe', 'sme']: raise Exception("Unknown radiative transfer code: %s" % (code)) reference_list_filename = output_dirname + "/parameters.tsv" if to_resolution is not None: reference_grid_filename = output_dirname + "/convolved_grid_%i.fits.gz" % to_resolution fits_dir = os.path.join(output_dirname, "grid/") mkdir_p(fits_dir) if steps: steps_fits_dir = os.path.join(output_dirname, "steps/") mkdir_p(steps_fits_dir) import dill # To allow pickle of lambda functions (e.g., one element in modeled_layers_pack) import pickle pickled_modeled_layers_pack = pickle.dumps(modeled_layers_pack) # For code != "grid", ranges are always in position 7 (for grid it would be in position 8) valid_ranges = modeled_layers_pack[7] teff_range = valid_ranges['teff'] logg_range = valid_ranges['logg'] MH_range = valid_ranges['MH'] alpha_range = valid_ranges.get( 'alpha', (-1.5, 1.5) ) # Default (0.,) if 'alpha' is not a free parameter for atmosphere interpolation vmic_range = valid_ranges.get( 'vmic', (0.0, 50.) ) # Default (0.,) if 'vmic' is not a free parameter for atmosphere interpolation # Parallelization pool if number_of_processes == 1: pool = None else: pool = Pool(number_of_processes) # Create grid binary file elapsed = 0 # seconds num_ref_spec = len(ranges) num_spec = num_ref_spec * 9 # Reference + 8 variations in Teff, logg, MH, alpha, vmic, vmac, vsini, limb darkening coeff i = 0 for teff, logg, MH, alpha, vmic in ranges: if vmic is None: vmic = estimate_vmic(teff, logg, MH) vmac = 0.0 # This can be modified after synthesis if needed vsini = 0.0 # This can be modified after synthesis if needed limb_darkening_coeff = 0.00 # This can be modified after synthesis if needed resolution = 0 # This can be modified after synthesis if needed is_step = False if not valid_atmosphere_target(modeled_layers_pack, { 'teff': teff, 'logg': logg, 'MH': MH, 'alpha': alpha }): raise Exception( "Target parameters out of the valid ranges: teff={} logg={} MH={} alpha={}" .format(teff, logg, MH, alpha)) points = [ (teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), ] if steps: is_step = True new_teff = teff + Constants.SYNTH_STEP_TEFF if teff + Constants.SYNTH_STEP_TEFF <= teff_range[ -1] else teff - Constants.SYNTH_STEP_TEFF new_logg = logg + Constants.SYNTH_STEP_LOGG if logg + Constants.SYNTH_STEP_LOGG <= logg_range[ -1] else logg - Constants.SYNTH_STEP_LOGG new_MH = MH + Constants.SYNTH_STEP_MH if MH + Constants.SYNTH_STEP_MH <= MH_range[ -1] else MH - Constants.SYNTH_STEP_MH new_alpha = alpha + Constants.SYNTH_STEP_ALPHA if alpha + Constants.SYNTH_STEP_ALPHA <= alpha_range[ -1] else alpha - Constants.SYNTH_STEP_ALPHA new_vmic = vmic + Constants.SYNTH_STEP_VMIC if vmic + Constants.SYNTH_STEP_VMIC <= vmic_range[ -1] else vmic - Constants.SYNTH_STEP_VMIC # For each reference point, calculate also the variations that iSpec will perform in the first iteration points += [ # Final unconvolved spectra where vmic/vmac are free and do not follow vmic/vmac empirical relations (new_teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, new_logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, new_MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, MH, new_alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, MH, alpha, new_vmic, vmac, vsini, limb_darkening_coeff, is_step), ] points += [ # Final unconvolved spectra where vmic is not free and does follow vmic empirical relations (new_teff, logg, MH, alpha, estimate_vmic(new_teff, logg, MH), vmac, vsini, limb_darkening_coeff, is_step), (teff, new_logg, MH, alpha, estimate_vmic(teff, new_logg, MH), vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, new_MH, alpha, estimate_vmic(teff, logg, new_MH), vmac, vsini, limb_darkening_coeff, is_step), ] for j, (teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step) in enumerate(points): if is_step: filename_out = steps_fits_dir + "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff) + ".fits.gz" else: filename_out = fits_dir + "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff) + ".fits.gz" if os.path.exists(filename_out): print("Skipping", teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, "already computed") continue if pool is None: if sys.platform == "win32": # On Windows, the best timer is time.clock() default_timer = time.clock else: # On most other platforms the best timer is time.time() default_timer = time.time lock = FileLock(filename_out + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is computing this spectrum, do not continue print("Skipping", teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, "already locked") continue try: tcheck = default_timer() # Validate parameters __generate_synthetic_fits(filename_out, wavelengths, segments, teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, resolution, pickled_modeled_layers_pack, atomic_linelist, isotopes, solar_abundances, code=code, use_molecules=use_molecules, tmp_dir=tmp_dir, locked=True) elapsed = default_timer() - tcheck print( "-----------------------------------------------------" ) print("Remaining time:") print("\t", (num_spec - i) * elapsed, "seconds") print("\t", (num_spec - i) * (elapsed / 60), "minutes") print("\t", (num_spec - i) * (elapsed / (60 * 60)), "hours") print("\t", (num_spec - i) * (elapsed / (60 * 60 * 24)), "days") print( "-----------------------------------------------------" ) finally: lock.release() else: pool.apply_async(__generate_synthetic_fits, [ filename_out, wavelengths, segments, teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, resolution, pickled_modeled_layers_pack, atomic_linelist, isotopes, solar_abundances ], kwds={ 'code': code, 'use_molecules': use_molecules, 'tmp_dir': tmp_dir, 'locked': False }) i += 1 if pool is not None: pool.close() pool.join() # Create parameters.tsv reference_list = Table() if len(np.unique(ranges[['logg', 'MH', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_teff', dtype=int)) else: reference_list.add_column(Column(name='teff', dtype=int)) if len(np.unique(ranges[['teff', 'MH', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_logg', dtype=float)) else: reference_list.add_column(Column(name='logg', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_MH', dtype=float)) else: reference_list.add_column(Column(name='MH', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'MH', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_alpha', dtype=float)) else: reference_list.add_column(Column(name='alpha', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'MH', 'alpha']])) == len(ranges): reference_list.add_column(Column(name='fixed_vmic', dtype=float)) else: reference_list.add_column(Column(name='vmic', dtype=float)) reference_list.add_column(Column(name='filename', dtype='|S100')) for teff, logg, MH, alpha, vmic in ranges: # Only use the first spectra generated for each combination zero_vmac = 0.0 zero_vsini = 0.0 zero_limb_darkening_coeff = 0.00 reference_filename_out = "./grid/{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, zero_vmac, zero_vsini, zero_limb_darkening_coeff) + ".fits.gz" reference_list.add_row( (int(teff), logg, MH, alpha, vmic, reference_filename_out)) if not os.path.exists(reference_list_filename): lock = FileLock(reference_list_filename + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is writing this file, do not continue print("Skipping", reference_list_filename, "already locked") else: try: ascii.write(reference_list, reference_list_filename, delimiter='\t', overwrite=True) print("Written", reference_list_filename) finally: lock.release() if to_resolution is not None: if not os.path.exists(reference_grid_filename): lock = FileLock(reference_grid_filename + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is computing this spectrum, do not continue print("Skipping", reference_grid_filename, "already locked") else: try: reference_grid = None complete_reference_list = Table() complete_reference_list.add_column( Column(name='teff', dtype=int)) complete_reference_list.add_column( Column(name='logg', dtype=float)) complete_reference_list.add_column( Column(name='MH', dtype=float)) complete_reference_list.add_column( Column(name='alpha', dtype=float)) complete_reference_list.add_column( Column(name='vmic', dtype=float)) complete_reference_list.add_column( Column(name='vmac', dtype=float)) complete_reference_list.add_column( Column(name='vsini', dtype=float)) complete_reference_list.add_column( Column(name='limb_darkening_coeff', dtype=float)) for teff, logg, MH, alpha, vmic in ranges: # Only use the first spectra generated for each combination zero_vmac = 0.0 zero_vsini = 0.0 zero_limb_darkening_coeff = 0.00 vmac = estimate_vmac(teff, logg, MH) vsini = 1.6 # Sun limb_darkening_coeff = 0.6 reference_filename_out = "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, zero_vmac, zero_vsini, zero_limb_darkening_coeff) + ".fits.gz" if not os.path.exists(fits_dir + reference_filename_out): continue complete_reference_list.add_row( (int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff)) # Spectra in the grid is convolved to the specified resolution for fast comparison print("Quick grid:", reference_filename_out) spectrum = read_spectrum(fits_dir + reference_filename_out) segments = None vrad = (0, ) spectrum['flux'] = apply_post_fundamental_effects(spectrum['waveobs'], spectrum['flux'], segments, \ macroturbulence=vmac, vsini=vsini, \ limb_darkening_coeff=limb_darkening_coeff, R=to_resolution, vrad=vrad) if reference_grid is None: reference_grid = spectrum['flux'] else: reference_grid = np.vstack( (reference_grid, spectrum['flux'])) if len(ranges) == len(complete_reference_list): # Generate FITS file with grid for fast comparison primary_hdu = fits.PrimaryHDU(reference_grid) wavelengths_hdu = fits.ImageHDU(wavelengths, name="WAVELENGTHS") params_bintable_hdu = fits.BinTableHDU( complete_reference_list.as_array(), name="PARAMS") fits_format = fits.HDUList([ primary_hdu, wavelengths_hdu, params_bintable_hdu ]) fits_format.writeto(reference_grid_filename, overwrite=True) print("Written", reference_grid_filename) finally: lock.release()
def test_write_bintable(self): """Test writing binary tables to FITS. """ from ..io.util import write_bintable, fitsheader # # Input: Table # hdr = fitsheader(dict(A=1, B=2)) hdr['C'] = ('BLAT', 'FOO') data = Table() data['X'] = [1, 2, 3] data['Y'] = [3, 4, 5] write_bintable(self.testfile, data, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: ndarray # hdr = dict(A=1, B=2) data = data.as_array() write_bintable(self.testfile, data, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: dictionary # hdr = dict(A=1, B=2) d = dict(X=np.array([1, 2, 3]), Y=np.array([3, 4, 5])) write_bintable(self.testfile, d, header=hdr) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) os.remove(self.testfile) # # Input: Table with column comments. # hdr = fitsheader(dict(A=1, B=2)) hdr['C'] = ('BLAT', 'FOO') data = Table() data['X'] = [1, 2, 3] data['Y'] = [3, 4, 5] write_bintable(self.testfile, data, header=hdr, comments={ 'X': 'This is X', 'Y': 'This is Y' }, units={ 'X': 'mm', 'Y': 'mm' }) # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True) self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) self.assertEqual(newhdr['TTYPE1'], 'X') self.assertEqual(newhdr.comments['TTYPE1'], 'This is X') self.assertEqual(newhdr['TTYPE2'], 'Y') self.assertEqual(newhdr.comments['TTYPE2'], 'This is Y') self.assertEqual(newhdr['TUNIT1'], 'mm') self.assertEqual(newhdr.comments['TUNIT1'], 'X units') self.assertEqual(newhdr['TUNIT2'], 'mm') self.assertEqual(newhdr.comments['TUNIT2'], 'Y units') # # Input: Table with no EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr) # # Input: Table with EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR') # # Standard suite of table tests. # result, newhdr = fits.getdata(self.testfile, header=True, extname='FOOBAR') self.assertEqual(sorted(result.dtype.names), sorted(data.dtype.names)) for colname in data.dtype.names: self.assertTrue(np.all(result[colname] == data[colname]), '{} data mismatch'.format(colname)) # self.assertEqual(newhdr.comments['C'], 'FOO') for key in hdr.keys(): self.assertIn(key, newhdr) self.assertIn('DATASUM', newhdr) self.assertIn('CHECKSUM', newhdr) # # Input: Table with existing EXTNAME, existing file # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR') # # Input: Table with EXTNAME, existing file, overwrite # write_bintable(self.testfile, data, header=hdr, extname='FOOBAR', clobber=True)
def __parse_result(self, row): """ Used to turn the catolog entries into dictionaries of information. Input ------ row: astropy row or single entry table The table to convert. Output ------ data: dic [keys=("Name","Otype","ra","dec","Distance","Mag","Sptype","Redshift","Luminosity") """ # Attempt to convert a row to a table (tables can be converted to arrays, but rows not) as_table = Table(row) # Extract information to single variables [ main, name, ids, idngc, idm, idhd, ra, dec, otype, sptype, plx, redshift, mag, dist, distu, distmeth ] = as_table.as_array()[0] # Make a string of all valid names usenames = "" for i in [name, idm, idngc, idhd]: if i != '' and i != 0: usenames += i + ", " data = {"Name": usenames[:-2]} # Go through the list, check if variables have information, and add them to dictionary if # information is found if mag != None: data["Mag"] = "%.3f" % mag if redshift > 0.001: data["Redshift"] = "%.3f" % redshift if otype != None: data["Otype"] = otype else: data["Otype"] = "Unknown" if sptype != '': data["Sptype"] = str(sptype) if plx > 0: # Distance calculations from parallax data["Plx"] = str(plx) distpc = 1000. / plx distly = distpc * 3.2616 data["Distance"] = "%.2f pc, %.2f ly" % (distpc, distly) if redshift != None: # Distance estimates from redshift data["redshift"] = redshift if plx > 0 and mag != None: # Luminosity estimates Msun = 4.74 absmag = mag - 5 * (np.log10(distpc) - 1) L = 10**(1. / 2.5 * (Msun - absmag)) data["Luminosity"] = "%.2f" % L if dist > 0: data["Distance"] = "%.2f %s" % (dist, distu) # Add right ascencion and declination information data["ra"] = "%s hrs" % ra data["dec"] = "%s deg" % dec data["celcoor"] = [utils.hourangle(ra), utils.degree(dec)] if "location" in dir(self): loc = self.location if "home" in dir(self): loc = self.home altaz = self.Transform.cel2altaz(data["celcoor"], loc)[0] data["altaz"] = altaz data["az"] = "%.2f deg" % altaz[0] data["alt"] = "%.2f deg" % altaz[1] # This is for a string with all of the information. Goes down the list, if a key exists, adds # infomation to string. Useful for displaying everything easily. order = [ "Name", "Otype", "ra", "dec", "az", "alt", "Mag", "Distance", "Sptype", "Luminosity", "Redshift" ] outstring = '' for key in order: if key in data.keys(): outstring += key + ": " + data[key] + "\n" data["String"] = outstring return data
from astropy.io import fits from astropy.table import Table import numpy as np e_bins = 20 o_bins = 5 e_axis = np.logspace(-1, 2, e_bins + 1) o_axis = np.linspace(0, 3, o_bins + 1) effarea = np.ones([e_bins, o_bins]) * 3e5 table = Table( [[e_axis[:-1]], [e_axis[1:]], [o_axis[:-1]], [o_axis[1:]], [effarea]], names=('ENERG_LO', 'ENERG_HI', 'THETA_LO', 'THETA_HI', 'EFFAREA')) data = table.as_array() header = fits.Header() header['OBS_ID'] = 31415, 'Observation ID' header['LO_THRES'] = 0.1, 'Low energy threshold [TeV]' header['HI_THRES'] = 50, 'High energy threshold [TeV]' header[ 'HDUDOC'] = 'https://github.com/open-gamma-ray-astro/gamma-astro-data-formats', '' header['HDUVERS'] = '0.2', '' header['HDUCLASS'] = 'GADF', '' header['HDUCLAS1'] = 'RESPONSE', '' header['HDUCLAS2'] = 'EFF_AREA', '' header['HDUCLAS3'] = 'FULL-ENCLOSURE', '' header['HDUCLAS4'] = 'AEFF_2D', '' tbhdu = fits.BinTableHDU(data, header, name='EFFECTIVE AREA')
def makeBFLineCube(self, cfg_par): cubeletsDir = cfg_par['general']['cubeletsDir'] cubeDir = cfg_par['general']['cubeDir'] momDir = cfg_par['general']['momDir'] if cfg_par['bestFitSel']['BFcube']['rotationID'] == True: modelCube = fits.open(cfg_par['bestFitSel']['BFcube']['modelCube']) mdC = modelCube[0].data rotMoM = np.zeros([mdC.shape[1], mdC.shape[2]]) * np.nan hdul = fits.open(cfg_par['general']['outTableName']) tabGen = hdul['BININFO'].data residuals = hdul['Residuals_' + cfg_par['gFit']['modName']].data ancels = hdul['Ancels' + cfg_par['gFit']['modName']].data rotArr = np.empty(len(ancels['BIN_ID'])) bF = np.array(residuals['bestFit'], dtype=int) wave, xAxis, yAxis, pxSize, noiseBin, vorBinInfo, dataSpec = tP.openVorLineOutput( cfg_par, cfg_par['general']['outVorLineTableName'], cfg_par['general']['outVorSpectra']) lineInfo = tP.openLineList(cfg_par) lineInfoAll = lineInfo.copy() index = np.where(lineInfo['BFCUbe'] == 0) fltr = np.array(index)[0] lineInfo.remove_rows(list(fltr)) lineNameStr = str(lineInfo['Name'][0]) if '[' in lineNameStr: lineName = lineNameStr.replace("[", "") lineName = lineName.replace("]", "") lineName = lineName + str(int(lineInfo['Wave'][0])) else: lineName = lineNameStr + str(int(lineInfo['Wave'][0])) lineNameName = lineNameStr + str(int(lineInfo['Wave'][0])) lineNameStr = np.array([lineNameStr + str(int(lineInfo['Wave'][0]))], dtype='a16') lineNamesStrAll = np.empty(len(lineInfoAll), dtype='a16') for ii in range(0, len(lineInfoAll['ID'])): #for ii in range(0,1): lineNameStrAll = str(lineInfoAll['Name'][ii]) if '[' in lineNameStrAll: lineNameAll = lineNameStrAll.replace("[", "") lineNameAll = lineNameAll.replace("]", "") lineNameAll = lineNameAll + str(int(lineInfoAll['Wave'][ii])) else: lineNameAll = lineNameStrAll + str(int( lineInfoAll['Wave'][ii])) lineNamesStrAll[ii] = str(lineNameStrAll + str(int(lineInfoAll['Wave'][ii]))) indexLine = np.where(lineNamesStrAll == lineNameStr)[0] modName = 'BF' f = fits.open(cubeDir + 'fitCube_' + modName + '.fits') dd = f[0].data hh = f[0].header mom = fits.open(momDir + 'g2/mom0_tot-' + lineName + '.fits') mm = mom[0].data indexFltr = np.broadcast_to(np.isnan(mm), dd.shape) dd[indexFltr] = np.nan #lambdaMin = np.log(cfg_par['gFit']['lambdaMin']) #lambdaMax = np.log(cfg_par['gFit']['lambdaMax']) #idxMin = int(np.where(abs(wave-lambdaMin)==abs(wave-lambdaMin).min())[0]) #idxMax = int(np.where(abs(wave-lambdaMax)==abs(wave-lambdaMax).min())[0]) lambdaMin = np.log(cfg_par['gFit']['lambdaMin']) lambdaMax = np.log(cfg_par['gFit']['lambdaMax']) idxMin = int( np.where(abs(wave - lambdaMin) == abs(wave - lambdaMin).min())[0]) idxMax = int( np.where(abs(wave - lambdaMax) == abs(wave - lambdaMax).min())[0]) wave = wave[idxMin:idxMax] dd = dd[idxMin:idxMax, :, :] velRangeMin = cvP.vRadLambda( -cfg_par['bestFitSel']['BFcube']['velRange'][0], lineInfo['Wave'][0]) - lineInfo['Wave'][0] velRangeMax = cvP.vRadLambda( cfg_par['bestFitSel']['BFcube']['velRange'][1], lineInfo['Wave'][0]) - lineInfo['Wave'][0] waveMin = np.log(lineInfo['Wave'][0] - velRangeMin) waveMax = np.log(lineInfo['Wave'][0] + velRangeMax) idxMin1 = int( np.where(abs(wave - waveMin) == abs(wave - waveMin).min())[0]) idxMax1 = int( np.where(abs(wave - waveMax) == abs(wave - waveMax).min())[0]) wave = wave[idxMin1:idxMax1] waveAng = np.exp(wave) vel = cvP.lambdaVRad(waveAng, lineInfo['Wave'][0]) + float( cfg_par['general']['velsys']) dd = dd[idxMin1:idxMax1, :, :] fitCube = np.empty([dd.shape[0], dd.shape[1], dd.shape[2]]) fitCubeMask = np.zeros([dd.shape[0], dd.shape[1], dd.shape[2]]) fitCubeMD = np.zeros([dd.shape[0], dd.shape[1], dd.shape[2]]) fitCubeMaskInter = np.zeros([dd.shape[0], dd.shape[1], dd.shape[2]]) vecSumMap = np.zeros([dd.shape[1], dd.shape[2]]) * np.nan lenghtLineMap = np.zeros([dd.shape[1], dd.shape[2]]) * np.nan for i in range(0, len(ancels['BIN_ID'])): match_bin = np.where(tabGen['BIN_ID'] == ancels['BIN_ID'][i])[0] if bF[i] == 0: modName = 'g1' elif bF[i] == 1: modName = 'g2' for index in match_bin: if np.sum(~np.isnan(dd[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])])) != 0: result = load_modelresult( cfg_par['general']['runNameDir'] + 'models/' + modName + '/' + str(ancels['BIN_ID'][i]) + '_' + modName + '.sav') comps = result.eval_components() if modName == 'g1': fit = comps['g1ln' + str(indexLine[0]) + '_'] elif modName == 'g2': fit = comps['g1ln' + str(indexLine[0]) + '_'] + comps['g2ln' + str(indexLine[0]) + '_'] fitCube[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = fit[idxMin1:idxMax1] if cfg_par['bestFitSel']['BFcube']['rotationID'] == True: mdSpec = mdC[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] mdSpec[mdSpec != 0] = 1. mdSpec = np.flipud(mdSpec) fitCubeMD[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = mdSpec #centroid = ancels['centroid_'+lineName][i] #width = ancels['w80_'+lineName][i] fitSmall = fit[idxMin1:idxMax1] idxPeak = np.nanargmax(fitSmall) #print(idxPeak) idxLeft = int( np.where( abs(fitSmall[:idxPeak] - 10.) == abs(fitSmall[:idxPeak] - 10.).min())[0]) idxRight = int( np.where( abs(fitSmall[idxPeak:] - 10.) == abs(fitSmall[idxPeak:] - 10.).min())[0]) + idxPeak #print(idxLeft,idxRight) #velMin = centroid-(width/2.) #velMax = centroid+(width/2.) #indexVelMin = int(np.where(abs(vel-velMin)==abs(vel-velMin).min())[0]) #indexVelMax = int(np.where(abs(vel-velMax)==abs(vel-velMax).min())[0]) fitMask = np.zeros(len(fit[idxMin1:idxMax1])) fitMaskIntercect = np.zeros(len(fit[idxMin1:idxMax1])) #fitMask[indexVelMin:indexVelMax] = 1. fitMask[idxLeft:idxRight] = 1. lenghtLine = np.count_nonzero(fitMask == 1.) fitCubeMask[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = fitMask #print(fitMask,mdSpec) vecCount = np.where((fitMask == 1.) & (mdSpec == 1.)) fitMaskIntercect[vecCount] = 1. vecSum = np.count_nonzero(fitMaskIntercect == 1.) fitCubeMaskInter[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index] )] = fitMaskIntercect vecSumMap[int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = vecSum lenghtLineMap[ int(tabGen['PixY'][index]), int(tabGen['PixX'][index] )] = lenghtLine / 100. * cfg_par['bestFitSel'][ 'BFcube']['rotationPercent'] if vecSum > (lenghtLine / 100. * cfg_par['bestFitSel'] ['BFcube']['rotationPercent']): rotArr[i] = 1. rotMoM[int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = 1. else: rotArr[i] = 0. rotMoM[int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = 0. else: fitCube[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan fitCubeMask[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan fitCubeMD[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan fitCubeMaskInter[:, int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan vecSumMap[int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan lenghtLineMap[int(tabGen['PixY'][index]), int(tabGen['PixX'][index])] = np.nan waveAng = np.exp(wave) header = self.makeHeader(cfg_par, lineInfo['Wave'][0], hh, waveAng) outCubelet = cubeletsDir + str(lineNameName) + '_BF.fits' outCubeletMask = cubeletsDir + str(lineNameName) + '_BFMask.fits' outCubeletMaskfl = cubeletsDir + str( lineNameName) + '_BFMaskInter.fits' outCubeletMaskMD = cubeletsDir + str(lineNameName) + '_BFMD.fits' fits.writeto(outCubelet, np.flip(fitCube, axis=0), header, overwrite=True) fits.writeto(outCubeletMask, np.flip(fitCubeMask, axis=0), header, overwrite=True) fits.writeto(outCubeletMaskfl, np.flip(fitCubeMaskInter, axis=0), header, overwrite=True) fits.writeto(outCubeletMaskMD, np.flip(fitCubeMD, axis=0), header, overwrite=True) if cfg_par['bestFitSel']['BFcube']['rotationID'] == True: outMomRot = momDir + str(lineNameName) + '_RotMom.fits' outMomSum = momDir + str(lineNameName) + '_SumInter.fits' outMomLength = momDir + str(lineNameName) + '_LengthLine.fits' outMomDiff = momDir + str(lineNameName) + '_diffInter.fits' if 'CUNIT3' in header: del header['CUNIT3'] if 'CTYPE3' in header: del header['CTYPE3'] if 'CDELT3' in header: del header['CDELT3'] if 'CRVAL3' in header: del header['CRVAL3'] if 'CRPIX3' in header: del header['CRPIX3'] if 'NAXIS3' in header: del header['NAXIS3'] header['WCSAXES'] = 2 header['SPECSYS'] = 'topocent' header['BUNIT'] = 'Jy/beam' fits.writeto(outMomRot, rotMoM, header, overwrite=True) fits.writeto(outMomSum, vecSumMap, header, overwrite=True) fits.writeto(outMomLength, lenghtLineMap, header, overwrite=True) fits.writeto(outMomDiff, vecSumMap - lenghtLineMap, header, overwrite=True) t = Table(ancels) if 'RotMod' not in ancels.dtype.names: t.add_column(Column(rotArr, name='RotMod')) else: t.replace_column('RotMod', Column(rotArr, name='RotMod')) try: tt = Table(hdul['AncelsBF'].data) hdul['AncelsBF'] = fits.BinTableHDU(t.as_array(), name='AncelsBF') except KeyError as e: tt = fits.BinTableHDU.from_columns(t.as_array(), name='AncelsBF') hdul.append(tt) hdul.writeto(cfg_par['general']['outTableName'], overwrite=True) return
def importing_data(password, user, Obs_date, Special_id1, Vertex1, Vertex2, Vertex3, Vertex4, Instrument, Mode, lambdaMu, eta, pv, relative_reflectance): #check if wavelenght is too big try: for wav1 in lambdaMu: if wav1 > 29.9999: raise ValueError("Required wavelenght", wav1, "is too big. Max 29.9999 micron") except: if lambdaMu > 29.9999: raise ValueError("Required wavelenght", lambdaMu, "is too big. Max 29.9999 micron") # Open database connection db = pymysql.connect("localhost", "{0}".format(user), "{0}".format(password)) # prepare a cursor object using cursor() method and go to the right database cursor = db.cursor() cursor.execute("USE ISPY") #import url and make a table url, z, TYPE = Url_for_ISPY_SkyCoord.ISPY_ephemeris_inSkyCoord( 'JWST', Obs_date, Special_id1, Vertex1, Vertex2, Vertex3, Vertex4) #delete the old data sql3 = "DROP TABLE IF EXISTS {0} ".format(Special_id1) try: # Execute the SQL command cursor.execute(sql3) # Commit your changes in the database db.commit() except: raise ValueError( "Error: unable to delete data in the old table to update it") #get table name and create a table for this Specific ID table_name, brightness = Setup_Table_in_MySQL_to_Fill_in_Data.makeMySQLtable( Special_id1, password, user, Obs_date, Vertex1, Vertex2, Vertex3, Vertex4, Instrument, Mode, lambdaMu) Special_id = table_name #update the fact that the data has been deleted time1 = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + ' UTC' sql11 = "UPDATE INPUT_Table SET Status='NULL',Number_of_Asteroids_Detected=0, Time_Updated_UTC='{0}' where OBS_id='{1}'".format( time1, Special_id) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError("Unable to update the status for OBS_id:" + Special_id + " in the INPUT_Table.") #check if there are any data on the website if z == None: #No results #update Status in the input table to know if that line has been run already time1 = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + ' UTC' sql11 = "UPDATE INPUT_Table SET Status='UPDATED',Number_of_Asteroids_Detected=0, Time_Updated_UTC='{0}' where OBS_id='{1}'".format( time1, Special_id) #print (sql11) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError("Unable to update the status for OBS_id:" + Special_id + "in the INPUT_Table.") else: #get the table and the tablelines distant, tablelines = Create_ISPY_Table.make_table( z, 'SPK-ID', 'EXPLANATION', 1, 1) #make the table into lines to put in the database f = Table.as_array(distant) #make a string of JPL_SPKID values JPL_SPKID = [] #loop over all the rows in the table from ISPY for x in range(0, len(f)): JPL_SPKID.append(f[x][0]) #table columns tab_col = [ 'JPL_SPKID', 'IAU_Number', 'Name_designation', 'RA', 'DEC1', 'Amag', 'dRAcosD', 'dDEC_by_dt', 'CntDst', 'PsAng', 'Data_Arc', 'Nobs', 'SMAA_3sig', 'SMIA_3sig', 'Theta', 'Pixel_x', 'Pixel_y', 'Last_updated', table_name, 'OBS_id', Special_id, 'H', 'G', 'alpha', 'r', 'delta', 'eta', 'pv', 'Relative_reflectance' ] #add Horizons data and the brightness asteroid = str(f[x][2]).strip("(") asteroid = str(asteroid).strip(")") horizons, V = Calculate_Brightness.calculate_brightness( lambdaMu, eta, pv, Obs_date, asteroid, relative_reflectance) #get the reflectance brightness reflectance = Ref_Brightness.Reflectance(relative_reflectance, V, lambdaMu) #add brightness values together Brightness = [] try: if horizons[8] == []: Brightness = 'empty' else: for xx in range(0, len(reflectance)): Brightness.append(horizons[8][xx] + reflectance[xx]) except: if horizons[8] == []: Brightness = 'empty' else: Brightness = horizons[8] + reflectance #input new data sql1 = "INSERT INTO {0}(OBS_id,JPL_SPKID,Name_designation,\ RA ,\ DEC1) \ VALUES ('{2}',{1[0]},'{1[2]}','{1[3]}','{1[4]}')".format( tab_col[18], f[x], tab_col[20]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError("Unable to import into table OBS_id:" + Special_id) #loop over all the columns in f(add data from ISPY) for y in range(0, len(tab_col) - 14): if y == 0 or y == 2 or y == 3 or y == 4: sql1 = 0 elif ma.getmask(f[x][y]) == True: # Prepare SQL query to INSERT a record into the database. sql1 = 0 elif f[x][y] == 'NA': # Prepare SQL query to INSERT a record into the database. sql1 = 0 else: sql1 = "UPDATE {0} SET {1}='{2}'\ where {3}={4}".format(tab_col[18], tab_col[y], f[x][y], tab_col[0], f[x][0]) #print (x,y,f[x][y]) try: # Execute the SQL command cursor.execute(sql1) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError( "Unable to import into table OBS_id:" + Special_id) #loop over all the columns in horizons (add data from Horizons and brightness) for yy in range(21, 29): sql3 = "UPDATE {0} SET {1}='{2}'\ where {3}={4}".format(tab_col[18], tab_col[yy], horizons[yy - 21], tab_col[0], f[x][0]) #print (sql3) try: # Execute the SQL command cursor.execute(sql3) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError("Unable to import into table OBS_id:" + Special_id) #print (horizons) try: for lam in range(0, len(lambdaMu)): if horizons[8] == []: sql4 = "Not possible to calculate brightness" else: #add values for brightness sql4 = "UPDATE {0} SET `{1}`='{2}'\ where {3}={4}".format(tab_col[18], brightness[lam], Brightness[lam], tab_col[0], f[x][0]) #print (sql4) try: # Execute the SQL command cursor.execute(sql4) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError( "Unable to import into table OBS_id:" + Special_id) except: if horizons[8] == []: sql4 = "Not possible to calculate brightness" else: #add values for brightness sql4 = "UPDATE {0} SET `{1}`='{2}'\ where {3}={4}".format(tab_col[18], brightness[0], Brightness, tab_col[0], f[x][0]) #print (sql4) try: # Execute the SQL command cursor.execute(sql4) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError( "Unable to import into table OBS_id:" + Special_id) #check if the number of rows in f and in MySQL is the same (so if all the data has been imported) sql2 = "SELECT COUNT(*) FROM {0}".format(tab_col[18]) try: # Execute the SQL command cursor.execute(sql2) # Fetch all the rows in a list of lists. result2 = cursor.fetchone() # Now check fetched result if result2[0] == len(f): print('For OBS ID ', Special_id, ': All the data have been recorded into the database.') #update Status in the input table to know if that line has been run already time1 = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + ' UTC' sql11 = "UPDATE INPUT_Table SET Status='UPDATED',Number_of_Asteroids_Detected={1}, Time_Updated_UTC='{2}' where OBS_id='{0}'".format( tab_col[18], result2[0], time1) #print (sql11) try: # Execute the SQL command cursor.execute(sql11) # Commit your changes in the database db.commit() except: # Rollback in case there is any error db.rollback() raise ValueError( "Unable to update the status for OBS_id:" + Special_id + "in the INPUT_Table.") else: print( 'For OBS ID ', Special_id, ': The number of asteroids in the database is not the same as from ISPY' ) except: raise ValueError( 'For OBS ID ', Special_id, ": Error: unable to fetch data to check if all the data have been recorded into the database." ) # disconnect from server db.close() return Special_id1
def aper_summary(self, gal_only=False, output=False): """Get all the stellar mass, age, metallicity, and velocity dispersion profiles. Parameters ---------- gal_only: bool, optional Only provide summary of the whole galaxy. Default: False. output : bool, optional Return the `maper` array when True. Default: False Configuration Parameters ------------------------ Can be found in `self.config`: subpix : int, optional Subpixel sampling factor. Default is 5. """ aper_sum = Table() # add expected values aper_sum.add_column(Column(data=self.rad_inn, name='rad_inn')) aper_sum.add_column(Column(data=self.rad_out, name='rad_out')) aper_sum.add_column(Column(data=self.rad_mid, name='rad_mid')) # aperture mass profiles if there and add to aper_sum table if 'map_star_rho_insitu_{}'.format( self.proj ) in self.hdf5_values and 'map_star_rho_exsitu_{}'.format( self.proj) in self.hdf5_values: self.maper('gal') aper_sum.add_column(Column(data=self.maper_gal, name='maper_gal')) if not gal_only: self.maper('ins') self.maper('exs') aper_sum.add_column( Column(data=self.maper_ins, name='maper_ins')) aper_sum.add_column( Column(data=self.maper_exs, name='maper_exs')) # aperture age profiles if there and add to aper_sum table if 'map_star_age_insitu_{}'.format( self.proj ) in self.hdf5_values and 'map_star_age_exsitu_{}'.format( self.proj) in self.hdf5_values: self.aprof('age', 'gal', return_mass=True) aper_sum.add_column( Column(data=self.age_prof_gal['prof_w'], name='age_gal_w')) aper_sum.add_column( Column(data=self.age_prof_gal['prof'], name='age_gal')) aper_sum.add_column( Column(data=self.age_prof_gal['flag'], name='age_gal_flag')) aper_sum.add_column( Column(data=self.age_prof_gal['mass'], name='mprof_gal')) if not gal_only: self.aprof('age', 'ins', return_mass=True) self.aprof('age', 'exs', return_mass=True) aper_sum.add_column( Column(data=self.age_prof_ins['prof_w'], name='age_ins_w')) aper_sum.add_column( Column(data=self.age_prof_ins['prof'], name='age_ins')) aper_sum.add_column( Column(data=self.age_prof_ins['flag'], name='age_ins_flag')) aper_sum.add_column( Column(data=self.age_prof_ins['mass'], name='mprof_ins')) aper_sum.add_column( Column(data=self.age_prof_exs['prof_w'], name='age_exs_w')) aper_sum.add_column( Column(data=self.age_prof_exs['prof'], name='age_exs')) aper_sum.add_column( Column(data=self.age_prof_exs['flag'], name='age_exs_flag')) aper_sum.add_column( Column(data=self.age_prof_exs['mass'], name='mprof_exs')) # Aperture metallicity profiles if there and add to aper_sum table if 'map_star_metallicity_insitu_{}'.format( self.proj ) in self.hdf5_values and 'map_star_metallicity_exsitu_{}'.format( self.proj) in self.hdf5_values: self.aprof('met', 'gal') aper_sum.add_column( Column(data=self.met_prof_gal['prof_w'], name='met_gal_w')) aper_sum.add_column( Column(data=self.met_prof_gal['prof'], name='met_gal')) aper_sum.add_column( Column(data=self.met_prof_gal['flag'], name='met_gal_flag')) if not gal_only: self.aprof('met', 'ins') self.aprof('met', 'exs') aper_sum.add_column( Column(data=self.met_prof_ins['prof_w'], name='met_ins_w')) aper_sum.add_column( Column(data=self.met_prof_ins['prof'], name='met_ins')) aper_sum.add_column( Column(data=self.met_prof_ins['flag'], name='met_ins_flag')) aper_sum.add_column( Column(data=self.met_prof_exs['prof_w'], name='met_exs_w')) aper_sum.add_column( Column(data=self.met_prof_exs['prof'], name='met_exs')) aper_sum.add_column( Column(data=self.met_prof_exs['flag'], name='met_exs_flag')) # Aperture velocity dispersion profiles if there and add to aper_sum table if 'map_star_sigma_insitu_{}'.format( self.proj ) in self.hdf5_values and 'map_star_sigma_exsitu_{}'.format( self.proj) in self.hdf5_values and 'map_star_sigma_{}'.format( self.proj) in self.hdf5_values: self.aprof('sigma', 'gal') aper_sum.add_column( Column(data=self.sigma_prof_gal['prof_w'], name='sigma_gal_w')) aper_sum.add_column( Column(data=self.sigma_prof_gal['prof'], name='sigma_gal')) aper_sum.add_column( Column(data=self.sigma_prof_gal['flag'], name='sigma_gal_flag')) if not gal_only: self.aprof('sigma', 'ins') self.aprof('sigma', 'exs') aper_sum.add_column( Column(data=self.sigma_prof_ins['prof_w'], name='sigma_ins_w')) aper_sum.add_column( Column(data=self.sigma_prof_ins['prof'], name='sigma_ins')) aper_sum.add_column( Column(data=self.sigma_prof_ins['flag'], name='sigma_ins_flag')) aper_sum.add_column( Column(data=self.sigma_prof_exs['prof_w'], name='sigma_exs_w')) aper_sum.add_column( Column(data=self.sigma_prof_exs['prof'], name='sigma_exs')) aper_sum.add_column( Column(data=self.sigma_prof_exs['flag'], name='sigma_exs_flag')) setattr(self, 'aper_sum', aper_sum.as_array()) if output: return aper_sum
print('Excluding columns not needed for proseco') excludes = ['PLX', 'PLX_ERR', 'PLX_CATID', 'ACQQ1', 'ACQQ2', 'ACQQ3', 'ACQQ4', 'ACQQ5', 'ACQQ6', 'XREF_ID1', 'XREF_ID2', 'XREF_ID3', 'XREF_ID4', 'XREF_ID5', 'RSV4', 'RSV5', 'RSV6', 'POS_CATID', 'PM_CATID', 'MAG', 'MAG_ERR', 'MAG_BAND', 'MAG_CATID', 'COLOR1_ERR', 'C1_CATID', # Keep color1, 2, 3 'COLOR2_ERR', 'C2_CATID', 'RSV2', 'VAR_CATID'] names = [name for name in stars.dtype.names if name not in excludes] print('Dtype before excluding:\n', stars.dtype) stars = Table([stars[name] for name in names], names=names, copy=False) stars = stars.as_array() print('Dtype after excluding:\n', stars.dtype) print('Sorting on Dec and re-ordering') idx = np.argsort(stars['DEC']) stars = stars.take(idx) print('Creating miniagasc.h5 file') rootname = 'proseco_agasc' if args.proseco else 'miniagasc' filename = '{}_{}.h5'.format(rootname, args.version) table_desc, bo = tables.descr_from_dtype(stars.dtype) minih5 = tables.open_file(filename, mode='w') minitbl = minih5.create_table('/', 'data', table_desc, title='AGASC {}'.format(num_version)) print('Appending stars to {} file'.format(filename))