def speclib_split(planfile,amsplit=True,cmsplit=True,nmsplit=True,oasplit=True,vtsplit=True,el='') :
""" Make a bunch of individual plan files from master, splitting [alpha/M],[C/M],[N/M],vt
"""
# read master plan file
print('splitting: ', planfile)
p=yaml.safe_load(open(planfile+'.yml','r'))
# some cards removed in split par files
p.pop('npart',None)
p.pop('npca',None)
p.pop('vmsuffix',None)
if el is not '' : p['elem'] = el
else: p.pop('elem',None)
# make specdir the full path relative to $APOGEE_SPECLIB/synth
if int(p['solarisotopes']) == 1 : isodir='solarisotopes'
elif int(p['solarisotopes']) == 2 : isodir='giantisotopes'
#for key in ['synthcode','atmos','specdir','linelist','config'] :
# p[key] = p[key].decode().strip("'")
p['specdir'] = p['synthcode']+'/'+p['atmos']+'/'+isodir+'/'+p['specdir']
# get ranges in [alpha/M], [C/M], [N/M], and vt
if amsplit :
amrange=spectra.vector(p['am0'],p['dam'],p['nam'])
p['nam'] = 1
else :
amrange = [0]
if cmsplit :
cmrange=spectra.vector(p['cm0'],p['dcm'],p['ncm'])
p['ncm'] = 1
else :
cmrange = [0]
if nmsplit :
nmrange=spectra.vector(p['nm0'],p['dnm'],p['nnm'])
p['nnm'] = 1
else :
nmrange = [0]
if oasplit :
try :
oarange=spectra.vector(p['oa0'],p['doa'],p['noa'])
except :
oasplit=False
oarange=[0.]
p['noa'] = 1
else :
oarange = [0]
if int(p['vmicrofit']) == 0 :
vtrange=spectra.vector(p['vt0'],p['dvt'],p['nvt'])
p.pop('vt0')
p.pop('dvt')
p.pop('nvt')
else :
vtrange = [0]
vtsplit = False
# loop through all and make individual plan files
dw = float(p['dw'])
for am in amrange :
if amsplit : p['am0'] = float(am)
for cm in cmrange :
if cmsplit : p['cm0'] = float(cm)
for nm in nmrange :
if nmsplit : p['nm0'] = float(nm)
for oa in oarange :
if oasplit : p['oa0'] = float(oa)
for vt in vtrange :
# vmicro handled differently
if int(p['vmicrofit']) == 0 :
p['vmicro'] = [float(10.**vt)]
# special handling for dw
if np.isclose(dw,-1.) :
if p['vmicro'] < 3.99 : p['dw'] = 0.05
else : p['dw'] = 0.10
suffix=''
if amsplit : suffix+='a'+atmos.cval(am)
if cmsplit : suffix+='c'+atmos.cval(cm)
if nmsplit : suffix+='n'+atmos.cval(nm)
if oasplit : suffix+='o'+atmos.cval(oa)
if vtsplit : suffix+='v'+atmos.cval(10.**vt)
p['name'] = suffix
with open(planfile+'_'+suffix+el+'.yml', 'w') as fp:
fp.write(yaml.dump(p,sort_keys=False,Dumper=Dumper))
def mergeholes(planfile='tgGK_180625.par',
dir='marcs/giantisotopes/tgGK_180625',
grid='GK',
fakehole=False,
cmrange=None,
nmrange=None,
vtrange=None,
apstar=False,
hard=None,
out='rbf_'):
# Read planfile and set output file name
if not os.path.isfile(planfile):
print('{:s} does not exist'.format(planfile))
return
p = yanny.yanny(planfile, np=True)
if dir is None:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/' + p[
'specdir'] + '/' if p.get('specdir') else './'
else:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/turbospec/' + dir + '/'
if cmrange is None: cmrange = spectra.vector(p['cm0'], p['dcm'], p['ncm'])
if nmrange is None: nmrange = spectra.vector(p['nm0'], p['dnm'], p['nnm'])
if vtrange is None:
try:
vtrange = 10.**spectra.vector(p['vt0'], p['dvt'], p['nvt'])
except:
vtrange = [float(p['vmicro'])]
if grid is None:
if p['specdir'].find('GK_') >= 0: grid = 'GK'
elif p['specdir'].find('M_') >= 0: grid = 'M'
elif p['specdir'].find('F_') >= 0: grid = 'F'
holefile = 'MARCS_' + grid + '_holefile.fits'
holes = fits.open(os.environ['APOGEE_SPECLIB'] +
'/atmos/marcs/MARCS_v3_2016/' + holefile)
for icm, cm in enumerate(cmrange):
for inm, nm in enumerate(nmrange):
for ivt, vt in enumerate(vtrange):
for iam, am in enumerate(
spectra.vector(p['am0'], p['dam'], p['nam'])):
print(am, cm, nm, vt)
file = ('a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(am), atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt))
filled = fits.open(out + file)[1].data
hcm = int(
round((cm - holes[0].header['CRVAL5']) /
holes[0].header['CDELT5']))
if hcm < 0:
#print('off carbon grid edge!',hcm)
hcm = 0
ham = int(
round((am - holes[0].header['CRVAL4']) /
holes[0].header['CDELT4']))
holes[0].data[hcm, ham, :, :, :] = filled
holes.writeto(out + holefile, overwrite=True)
def fill(planfile='tgGK_180625.par',
dir='marcs/giantisotopes/tgGK_180625',
cmrange=None,
nmrange=None,
vtrange=None,
grid='GK',
apstar=False,
threads=30,
fakehole=False,
out='rbf_',
r0=1.0):
""" routine to fill grid holes using RBF interpolation routine from Szabolcs
"""
# Read planfile and set output file name
if not os.path.isfile(planfile):
print('{:s} does not exist'.format(planfile))
return
p = yaml.safe_load(open(planfile, 'r'))
if p.get('r0'): r0 = float(p['r0'])
vmicrofit = int(p['vmicrofit']) if p.get('vmicrofit') else 0
vmicro = np.array(p['vmicro']) if p.get('vmicro') else 0.
# input directory
if dir is None:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/' + p[
'specdir'] + '/' if p.get('specdir') else './'
else:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/turbospec/' + dir + '/'
print('indir: ', indir)
if cmrange is None: cmrange = spectra.vector(p['cm0'], p['dcm'], p['ncm'])
if nmrange is None: nmrange = spectra.vector(p['nm0'], p['dnm'], p['nnm'])
if vtrange is None:
try:
vtrange = 10.**spectra.vector(p['vt0'], p['dvt'], p['nvt'])
except:
vtrange = vmicro
# get configuration for grid
# sizes of subgrids for RBF interpolation in [alpha/M], [M/H], logg, and Teff
teffsize = subgrid(int(p['nteff']))
amsize = subgrid(int(p['nam']))
mhsize = subgrid(int(p['nmh']))
loggsize = subgrid(int(p['nlogg']))
if grid is None:
if p['specdir'].find('GK_') >= 0: grid = 'GK'
elif p['specdir'].find('M_') >= 0: grid = 'M'
elif p['specdir'].find('F_') >= 0: grid = 'F'
# read holes file
holefile = 'MARCS_' + grid + '_holefile.fits'
holes = fits.open(os.environ['APOGEE_SPECLIB'] +
'/atmos/marcs/MARCS_v3_2016/' + holefile)[0]
# total number of frequencies, and pixels to use
if apstar:
prefix = ''
nfreq = aspcap.nw_chip.sum()
pix_apstar = aspcap.gridPix()
pix_aspcap = aspcap.gridPix(apStar=False)
else:
prefix = ''
file = (prefix + 'a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(0.), atmos.cval(cmrange[0]), atmos.cval(nmrange[0]),
atmos.cval(vtrange[0]))
grid = fits.open(indir + file)[0]
nfreq = grid.data.shape[-1]
# loop over [C/M], [N/M], and vmicro, and set up the subgrids for interpolation
nrbf = 0
for icm, cm in enumerate(cmrange):
hcm = int(round(
(cm - holes.header['CRVAL5']) / holes.header['CDELT5']))
if hcm < 0:
#print('off carbon grid edge!',hcm)
hcm = 0
for inm, nm in enumerate(nmrange):
for ivt, vt in enumerate(vtrange):
# load into grids of [alpha,mh,logg,teff,wave]
data = np.zeros([
int(p['nam']),
int(p['nmh']),
int(p['nlogg']),
int(p['nteff']), nfreq
],
dtype=np.float32)
for iam, am in enumerate(
spectra.vector(p['am0'], p['dam'], p['nam'])):
file = (prefix + 'a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(am), atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt))
grid = fits.open(indir + file)[0]
if apstar:
# select out ASPCAP grid wavelengths
for pasp, pap in zip(pix_aspcap, pix_apstar):
data[iam, :, :, :,
pasp[0]:pasp[1]] = grid.data[:, :, :,
pap[0]:pap[1]]
else:
data[iam, :, :, :, :] = grid.data
# within each of these grids, do rbf in subsets in parallel, so set up input for subgrids
pars = []
npars = 0
sam = 0
for nam in amsize:
smh = 0
for nmh in mhsize:
slogg = 0
for nlogg in loggsize:
steff = 0
for nteff in teffsize:
# get the subgrid of holes
# try to extend subgrid by one to avoid edges
am1, am2, ham1, ham2 = extend(
sam, sam + nam,
spectra.vector(p['am0'], p['dam'],
p['nam']),
spectra.fits2vector(holes.header, 4))
mh1, mh2, hmh1, hmh2 = extend(
smh, smh + nmh,
spectra.fits2vector(grid.header, 4),
spectra.fits2vector(holes.header, 3))
logg1, logg2, hlogg1, hlogg2 = extend(
slogg, slogg + nlogg,
spectra.fits2vector(grid.header, 3),
spectra.fits2vector(holes.header, 2))
teff1, teff2, hteff1, hteff2 = extend(
steff, steff + nteff,
spectra.fits2vector(grid.header, 2),
spectra.fits2vector(holes.header, 1))
# get range in holes file
ham = int(
round((float(p['am0']) +
sam * float(p['dam']) -
holes.header['CRVAL4']) /
holes.header['CDELT4']))
hmh = int(
round((grid.header['CRVAL4'] +
smh * grid.header['CDELT4'] -
holes.header['CRVAL3']) /
holes.header['CDELT3']))
hlogg = int(
round((grid.header['CRVAL3'] +
slogg * grid.header['CDELT3'] -
holes.header['CRVAL2']) /
holes.header['CDELT2']))
hteff = int(
round((grid.header['CRVAL2'] +
steff * grid.header['CDELT2'] -
holes.header['CRVAL1']) /
holes.header['CDELT1']))
nholes = len(
np.where(
holes.data[hcm, ham:ham + nam,
hmh:hmh + nmh, hlogg:hlogg +
nlogg, hteff:hteff +
nteff] > 0)[0])
# add "fake" holes if desired
if fakehole:
holes.data[hcm, ham + nam / 2,
hmh + nmh / 2,
hlogg + nlogg / 2,
hteff] += 100.
holes.data[hcm, ham + nam / 2,
hmh + nmh / 2, hlogg,
hteff + nteff / 2] += 100.
holes.data[hcm, ham + nam / 2, hmh,
hlogg + nlogg / 2,
hteff + nteff / 2] += 100.
holes.data[hcm, ham, hmh + nmh / 2,
hlogg + nlogg / 2,
hteff + nteff / 2] += 100.
# set up input for RBF: (name, data, holes)
name = out + 'c{:s}n{:s}v{:s}_{:02d}'.format(
atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt), npars)
print(name, am1, am2, mh1, mh2, logg1, logg2,
teff2, teff2, hcm, ham1, ham2, hmh1,
hmh2, hlogg1, hlogg2, hteff1, hteff2)
pars.append(
(name, r0, data[am1:am2, mh1:mh2,
logg1:logg2,
teff1:teff2, :],
np.squeeze(holes.data[hcm, ham1:ham2,
hmh1:hmh2,
hlogg1:hlogg2,
hteff1:hteff2])))
npars += 1
steff += nteff
slogg += nlogg
smh += nmh
sam += nam
# do the RBF in parallel for the subgrids
pool = mp.Pool(threads)
specs = pool.map_async(dorbf, pars).get()
pool.close()
pool.join()
# fill in the holes in the full 4D grid with the interpolated spectra that are returned
# but don't use the "extended" regions
ii = 0
sam = 0
for nam in amsize:
smh = 0
for nmh in mhsize:
slogg = 0
for nlogg in loggsize:
steff = 0
for nteff in teffsize:
am1, am2, ham1, ham2 = extend(
sam, sam + nam,
spectra.vector(p['am0'], p['dam'],
p['nam']),
spectra.fits2vector(holes.header, 4))
mh1, mh2, hmh1, hmh2 = extend(
smh, smh + nmh,
spectra.fits2vector(grid.header, 4),
spectra.fits2vector(holes.header, 3))
logg1, logg2, hlogg1, hlogg2 = extend(
slogg, slogg + nlogg,
spectra.fits2vector(grid.header, 3),
spectra.fits2vector(holes.header, 2))
teff1, teff2, hteff1, hteff2 = extend(
steff, steff + nteff,
spectra.fits2vector(grid.header, 2),
spectra.fits2vector(holes.header, 1))
# get range in holes file
ham = int(
round((float(p['am0']) +
sam * float(p['dam']) -
holes.header['CRVAL4']) /
holes.header['CDELT4']))
hmh = int(
round((grid.header['CRVAL4'] +
smh * grid.header['CDELT4'] -
holes.header['CRVAL3']) /
holes.header['CDELT3']))
hlogg = int(
round((grid.header['CRVAL3'] +
slogg * grid.header['CDELT3'] -
holes.header['CRVAL2']) /
holes.header['CDELT2']))
hteff = int(
round((grid.header['CRVAL2'] +
steff * grid.header['CDELT2'] -
holes.header['CRVAL1']) /
holes.header['CDELT1']))
nholes = len(
np.where(
holes.data[hcm, ham:ham + nam,
hmh:hmh + nmh, hlogg:hlogg +
nlogg, hteff:hteff +
nteff] > 0)[0])
name = out + 'c{:s}n{:s}v{:s}_{:02d}'.format(
atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt), ii)
print('loading: ', name, am1, am2, mh1, mh2,
logg1, logg2, teff2, teff2, hcm, ham1,
ham2, hmh1, hmh2, hlogg1, hlogg2, hteff1,
hteff2)
# replace data and holes with filled data
data[sam:sam+nam,smh:smh+nmh,slogg:slogg+nlogg,steff:steff+nteff,:] = \
specs[ii][0][sam-am1:sam-am1+nam,smh-mh1:smh-mh1+nmh,slogg-logg1:slogg-logg1+nlogg,steff-teff1:steff-teff1+nteff,:]
try:
holes.data[hcm,ham:ham+nam,hmh:hmh+nmh,hlogg:hlogg+nlogg,hteff:hteff+nteff] = \
specs[ii][1][ham-ham1:ham-ham1+nam,hmh-hmh1:hmh-hmh1+nmh,hlogg-hlogg1:hlogg-hlogg1+nlogg,hteff-hteff1:hteff-hteff1+nteff]
except:
pdb.set_trace()
ii += 1
steff += nteff
slogg += nlogg
smh += nmh
sam += nam
# write out the 4D grid across appropreiate output files (separate for each [alpha/m])
for iam, am in enumerate(
spectra.vector(p['am0'], p['dam'], p['nam'])):
file = ('a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(am), atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt))
grid = fits.open(indir + prefix + file)
if apstar:
# fill in apStar wavelengths
for pasp, pap in zip(pix_aspcap, pix_apstar):
grid[0].data[:, :, :,
pap[0]:pap[1]] = data[iam, :, :, :,
pasp[0]:pasp[1]]
else:
grid[0].data = data[iam, :, :, :, :]
grid[0].header.add_comment(
'holes filled using RBF interpolation')
grid[0].header.add_comment('APOGEE_VER:' +
os.environ['APOGEE_VER'])
grid[0].header.append(
('R0', r0, 'value of r0 used for RBF interpolation'))
ham = int(
round((am - holes.header['CRVAL4']) /
holes.header['CDELT4']))
# append the modified holes file for this subgrid
hout = fits.ImageHDU(
np.squeeze(holes.data[hcm, ham, :, :, :]))
for idim in range(1, 4):
spectra.add_dim(hout.header,
holes.header['CRVAL' + str(idim)],
holes.header['CDELT' + str(idim)], 1,
holes.header['CTYPE' + str(idim)],
idim)
new = fits.HDUList()
new.append(grid[0])
new.append(hout)
new.writeto(indir + out + file, overwrite=True)
def comp(planfile='tgGK_180625.par',
dir='marcs/giantisotopes/tgGK_180625',
grid='GK',
fakehole=False,
cmrange=None,
nmrange=None,
vtrange=None,
apstar=False,
hard=None,
out='rbf_'):
# Read planfile and set output file name
if not os.path.isfile(planfile):
print('{:s} does not exist'.format(planfile))
return
p = yanny.yanny(planfile, np=True)
# input directory
if dir is None:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/' + p[
'specdir'] + '/' if p.get('specdir') else './'
else:
indir = os.environ['APOGEE_SPECLIB'] + '/synth/turbospec/' + dir + '/'
if grid is None:
if p['name'].find('GK_'): grid = 'GK'
elif p['name'].find('M_'): grid = 'M'
elif p['name'].find('F_'): grid = 'F'
if cmrange is None: cmrange = spectra.vector(p['cm0'], p['dcm'], p['ncm'])
if nmrange is None: nmrange = spectra.vector(p['nm0'], p['dnm'], p['nnm'])
if vtrange is None:
try:
vtrange = 10.**spectra.vector(p['vt0'], p['dvt'], p['nvt'])
except:
vtrange = [float(p['vmicro'])]
if grid is None:
if p['specdir'].find('GK_') >= 0: grid = 'GK'
elif p['specdir'].find('M_') >= 0: grid = 'M'
elif p['specdir'].find('F_') >= 0: grid = 'F'
holefile = 'MARCS_' + grid + '_holefile.fits'
if fakehole: holes = fits.open(out + holefile)[0]
else:
holes = fits.open(os.environ['APOGEE_SPECLIB'] +
'/atmos/marcs/MARCS_v3_2016/' + holefile)[0]
if apstar:
prefix = ''
else:
prefix = ''
ii = 0
nx = 5
ny = 6
figs = []
for icm, cm in enumerate(cmrange):
for inm, nm in enumerate(nmrange):
for ivt, vt in enumerate(vtrange):
for iam, am in enumerate(
spectra.vector(p['am0'], p['dam'], p['nam'])):
print(am, cm, nm, vt)
file = ('a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(am), atmos.cval(cm), atmos.cval(nm),
atmos.cval(vt))
raw = fits.open(indir + prefix + file)[0].data
filled = fits.open(out + file)[0].data
#holes=fits.open(out+file)[1].data
for imh, mh in enumerate(
spectra.vector(p['mh0'], p['dmh'], p['nmh'])):
for ilogg, logg in enumerate(
spectra.vector(p['logg0'], p['dlogg'],
p['nlogg'])):
for iteff, teff in enumerate(
spectra.vector(p['teff0'], p['dteff'],
p['nteff'])):
hcm = int(
round((cm - holes.header['CRVAL5']) /
holes.header['CDELT5']))
if hcm < 0:
#print('off carbon grid edge!',hcm)
hcm = 0
ham = int(
round((am - holes.header['CRVAL4']) /
holes.header['CDELT4']))
hmh = int(
round((mh - holes.header['CRVAL3']) /
holes.header['CDELT3']))
hlogg = int(
round((logg - holes.header['CRVAL2']) /
holes.header['CDELT2']))
hteff = int(
round((teff - holes.header['CRVAL1']) /
holes.header['CDELT1']))
if ((fakehole
and abs(holes.data[hcm, ham, hmh, hlogg,
hteff]) > 99.) or
(not fakehole
and abs(holes.data[hcm, ham, hmh, hlogg,
hteff]) > 0)):
print(
cm, nm, vt, am, mh, logg, teff,
holes.data[hcm, ham, hmh, hlogg,
hteff])
lab = '{:6.2f}{:6.2f}{:6.2f}{:6.0f}{:7.2f}'.format(
am, mh, logg, teff,
holes.data[hcm, ham, hmh, hlogg,
hteff])
lab1 = '{:6.2f}{:6.2f}{:6.2f}'.format(
cm, nm, vt)
if hard:
if ii % (nx * ny) == 0:
fig, ax = plots.multi(nx,
ny,
hspace=0.001,
wspace=0.001,
figsize=(14,
8))
iy = ii % ny
ix = (ii % (nx * ny)) / ny
print(ii, iy, ix)
#ax[iy,ix].plot(raw[imh,ilogg,iteff,:]/np.nanmean(raw[imh,ilogg,iteff,:]))
ax[iy, ix].plot(filled[imh, ilogg,
iteff, :],
color='g')
ax[iy, ix].plot(
raw[imh, ilogg, iteff, :] /
np.nanmean(raw[imh, ilogg,
iteff, :]) /
filled[imh, ilogg, iteff, :] + 0.2)
ax[iy, ix].set_ylim([0.7, 1.35])
if not np.isclose(
holes.data[hcm, ham, hmh,
hlogg, hteff],
0.) and not np.isclose(
holes.data[hcm, ham, hmh,
hlogg, hteff],
-100.):
color = 'r'
else:
color = 'k'
ax[iy,
ix].text(0.01,
0.97,
lab,
transform=ax[iy,
ix].transAxes,
va='top',
fontsize='x-small',
color=color)
ax[iy,
ix].text(0.01,
0.9,
lab1,
transform=ax[iy,
ix].transAxes,
va='top',
fontsize='x-small',
color=color)
ii += 1
if ii % (nx * ny) == 0:
fig.savefig(
out +
'{:02d}'.format(ii /
(nx * ny)) +
'.png')
figs.append([
out +
'{:02d}'.format(ii /
(nx * ny)) +
'.png'
])
plt.close()
else:
plt.clf()
plt.plot(raw[imh, ilogg, iteff, :] /
np.nanmean(raw[imh, ilogg,
iteff, :]) -
0.2,
color='b')
plt.plot(filled[imh, ilogg, iteff, :],
color='g')
plt.plot(raw[imh, ilogg, iteff, :] /
np.nanmean(raw[imh, ilogg,
iteff, :]) /
filled[imh, ilogg, iteff, :] +
0.2,
color='r')
plt.ylim([0.7, 1.3])
plt.draw()
pdb.set_trace()
html.htmltab(figs, file=out + '.html')
def dopca(pars) :
""" do a single PCA decomposition for a limited number of pixels
Args:
pars : input tuple giving required input
"""
ipiece = pars[0]
showtime('start piece: '+str(ipiece))
p=pars[1]
wrange=pars[2]
w1=wrange[0]
w2=wrange[1]
npix=w2-w1
npiece=p['npiece']
npca=p['npca']
whiten=p['whiten']
indir=p['indir']
outfile=p['outfile']
outdir=p['outdir']
writeraw=p['writeraw']
rawsynth=p['rawsynth']
prefix=p['prefix']
# load data for this piece
pcadata=np.zeros([int(p['noa'])*int(p['nam'])*int(p['ncm'])*int(p['nnm'])*int(p['nvt'])*
int(p['nrot'])*int(p['nmh'])*int(p['nlogg'])*int(p['nteff']),npix],dtype=np.float32)
nmod=0
pix_apstar=aspcap.gridPix()
pix_aspcap=aspcap.gridPix(apStar=False)
if rawsynth:
wave=np.arange(15100.,17000.01,0.05)
nwave=len(wave)
else :
nwave=aspcap.nw_chip.sum()
for ioa,oa in enumerate(spectra.vector(p['oa0'],p['doa'],p['noa'])) :
for ivm,vm in enumerate(spectra.vector(p['vt0'],p['dvt'],p['nvt'])) :
for icm,cm in enumerate(spectra.vector(p['cm0'],p['dcm'],p['ncm'])) :
for inm,nm in enumerate(spectra.vector(p['nm0'],p['dnm'],p['nnm'])) :
for iam,am in enumerate(spectra.vector(p['am0'],p['dam'],p['nam'])) :
if int(p['noa']) == 1 and int(p['nvt']) >= 1 :
file=('a{:s}c{:s}n{:s}v{:s}.fits').format(
atmos.cval(am),atmos.cval(cm),atmos.cval(nm),atmos.cval(10**vm))
elif int(p['noa']) > 1 and int(p['nvt']) == 1 :
file=('a{:s}c{:s}n{:s}o{:s}.fits').format(
atmos.cval(am),atmos.cval(cm),atmos.cval(nm),atmos.cval(oa))
else :
file=('a{:s}c{:s}n{:s}.fits').format(
atmos.cval(am),atmos.cval(cm),atmos.cval(nm))
# read file and pack into ASPCAP grid size
sap=fits.open(indir+prefix+file)[0].data
sap=sap.reshape((int(p['nrot']),int(p['nmh']),int(p['nlogg']),int(p['nteff']),sap.shape[-1]))
s=np.zeros([int(p['nrot']),int(p['nmh']),int(p['nlogg']),int(p['nteff']),nwave])
for pasp,pap in zip(pix_aspcap,pix_apstar) :
s[:,:,:,:,pasp[0]:pasp[1]]=sap[:,:,:,:,pap[0]:pap[1]]
#s1=fits.open(indir+file)[1].data
#s2=fits.open(indir+file)[2].data
#s3=fits.open(indir+file)[3].data
for irot,rot in enumerate(spectra.vector(p['rot0'],p['drot'],p['nrot'])) :
for imh,mh in enumerate(spectra.vector(p['mh0'],p['dmh'],p['nmh'])) :
for ilogg,logg in enumerate(spectra.vector(p['logg0'],p['dlogg'],p['nlogg'])) :
for iteff,teff in enumerate(spectra.vector(p['teff0'],p['dteff'],p['nteff'])) :
#s=np.append(s1[imh,ilogg,iteff,:],s2[imh,ilogg,iteff,:])
#s=np.append(s,s3[imh,ilogg,iteff,:])
#s=sall[imh,ilogg,iteff,:]
if s[irot,imh,ilogg,iteff,w1:w2].sum() == 0. or not np.isfinite(np.nanmean(s[irot,imh,ilogg,iteff,:])) :
print('!!ZERO or NaN MODEL',am,cm,nm,vm,rot,mh,logg,teff,w1,w2)
s[irot,imh,ilogg,iteff,w1:w2] = 1.
# s/=np.nanmean(s)
# if len(np.where(np.isfinite(s) is False)[0]) > 0 : print(imh,ilogg,iteff,np.where(np.isfinite(s) is False) )
pcadata[nmod,:] = s[irot,imh,ilogg,iteff,w1:w2]/np.nanmean(s[irot,imh,ilogg,iteff,:])
nmod+=1
#del s1, s2, s3, s
del s
# do the PCA decomposition
if npca > 0 :
if p['incremental'] :
print('using incremental PCA')
pca = IncrementalPCA(n_components=npca,whiten=whiten,batch_size=1000)
else :
pca = PCA(n_components=npca,whiten=whiten)
print(pcadata.shape)
showtime('start pca: '+str(ipiece))
model=pca.fit_transform(pcadata)
print(pca.explained_variance_ratio_)
eigen = pca.components_
mean = pca.mean_
else :
pca = 0
eigen = 0.
mean = 0.
# write out uncompressed and compressed in binary to local file
showtime('start write: '+str(ipiece))
if writeraw: fraw=open(outdir+'/'+outfile+'_{:03d}_{:03d}_{:03d}.raw'.format(npiece,npca,ipiece),'wb')
if npca > 0 :
fpca=open(outdir+'/'+outfile+'_{:03d}_{:03d}_{:03d}.pca'.format(npiece,npca,ipiece),'wb')
feigen=open(outdir+'/'+outfile+'_{:03d}_{:03d}_{:03d}.eigen'.format(npiece,npca,ipiece),'wb')
for i in range(npca) : feigen.write(struct.pack('f'*npix,*eigen[i,:]))
feigen.write(struct.pack('f'*npix,*mean))
feigen.close()
for i in range(nmod) :
if writeraw: fraw.write(struct.pack('f'*npix,*pcadata[i,:]))
if npca > 0 : fpca.write(struct.pack('f'*npca,*model[i,:]))
if writeraw: fraw.close()
if npca > 0 : fpca.close()
showtime('done piece: '+str(ipiece))
del pca
del pcadata
return eigen,mean