def generateBoxPlot(self, setnames=None, variable=None, filename='output'):
if setnames is None:
print("Warning: no setnames provided. Assuming all elements and nodes in model.")
elif not isinstance(setnames, list):
setnames = [setnames]
if variable is None or not isinstance(variable, str):
raise TypeError(variable)
for s in setnames:
fig = plt.Figure()
ax = fig.add_subplot(111)
quartiles = []
medians = []
df1 = pd.DataFrame()
for k, v in self.element_sets[s].items():
weights = numpy_support.vtk_to_numpy(v.GetCellData().GetArray('Reference Volume'))
data = numpy_support.vtk_to_numpy(v.GetCellData().GetArray(variable))
# idx = np.argwhere(data > 0.1).ravel()
# weights = weights[idx]
weights /= np.max(weights)
# data = data[idx]
medians.append(quantile(data, weights, 0.5))
quartiles.append([quantile(data, weights, x) for x in [0.0, 0.25, 0.75, 1.0]])
df2 = pd.DataFrame({k: np.array(data, dtype="object")})
df1 = pd.concat([df1, df2], axis=1)
df1.boxplot(ax=ax, vert=False, grid=False)
ax.xaxis.tick_top()
ax.xaxis.set_label_position('top')
fig.savefig('{}.svg'.format(filename))
def open_filters(fdir, usecols=[0, 1]):
ff = RootDir + fdir + '/'
fil_ext = 'fil' if fdir == 'CFHTLS' else 'dat'
flist = glob.glob(ff + '*.' + fil_ext)
nfilters = len(flist)
print '%ld filters found for %s' % (nfilters, fdir)
if nfilters == 0:
raise ValueError('no filters found!')
filters = {} #'flist':flist}
filters_sorted = {} #'flist':flist}
fname = []
meanw = []
for i in range(nfilters):
fdata = np.loadtxt(flist[i], usecols=usecols)
filtername = string.rsplit(flist[i], '/', 1)[1].rsplit('.', 1)[0]
fname.append(filtername)
meanw.append(wp.quantile(fdata[:, 0], fdata[:, 1], 0.5))
filters.update({filtername: fdata})
filters.update({'filters': fname})
filters.update({'MeanWavelength': meanw})
wsort = np.argsort(filters['MeanWavelength'])
fnamelist = np.asarray(filters['filters'])[wsort]
filters_sorted.update({'filters': fnamelist})
for i in range(nfilters):
f_i = fnamelist[i]
filters_sorted.update({f_i: filters[f_i]})
filters_sorted.update(
{'MeanWavelength': np.asarray(filters['MeanWavelength'])[wsort]})
return filters_sorted
def test_weighted_median_3D(self):
arr1 = quantile(self.a3D, self.aw, 0.5)
arr2 = np.array([[ 43.66666667, 91., 35., 50., 23.],[30.66666667, 89., 75.66666667, 6., 48.33333333],
[49., 54.66666667, 16.33333333, 89.33333333, 26.33333333],
[63., 34., 37.33333333, 57.66666667, 82.], [34., 32., 29., 37., 51.33333333]])
#print(arr1, arr2)
nptest.assert_array_almost_equal(arr1, arr2)
def test_weighted_median_3D(self):
arr1 = quantile(self.a3D, self.aw, 0.5)
arr2 = np.array(
[[43.66666667, 91., 35., 50., 23.],
[30.66666667, 89., 75.66666667, 6., 48.33333333],
[49., 54.66666667, 16.33333333, 89.33333333, 26.33333333],
[63., 34., 37.33333333, 57.66666667, 82.],
[34., 32., 29., 37., 51.33333333]])
#print(arr1, arr2)
nptest.assert_array_almost_equal(arr1, arr2)
def calc_fit_power_law(delta_f_snr_bins=snr_stats_total):
snr_bins = delta_f_snr_bins_helper.get_log_snr_axis()
y_quantile = np.zeros_like(snr_bins)
y1 = delta_f_snr_bins_helper.get_delta_f_axis()
for i in range(50):
y_quantile[i] = weighted.quantile(y1, delta_f_snr_bins[i], .9)
mask = [np.logical_and(-0 < snr_bins, snr_bins < 3)]
masked_snr_bins = snr_bins[mask]
# print("x2:", masked_snr_bins)
fit_params = lmfit.Parameters()
fit_params.add('a', -2., min=-5, max=-1)
fit_params.add('b', 1., min=0.1, max=20.)
fit_params.add('c', 0.08, min=0, max=0.2)
fit_params.add('d', 3, min=-5, max=5)
fit_result = lmfit.minimize(fit_function, fit_params, kws={'data': y_quantile[mask], 'x': masked_snr_bins})
return fit_result, snr_bins, masked_snr_bins, y_quantile
def reduce_and_save(output_file, global_histogram, histogram,
group_parameters):
comm.Reduce([histogram, MPI.DOUBLE], [global_histogram, MPI.DOUBLE],
op=MPI.SUM,
root=0)
if comm.rank == 0:
# compute the median and add it to the npz file
ism_spec = np.zeros(shape=global_histogram.shape[1], dtype=np.double)
for i in range(ism_spec.size):
ism_spec[i] = weighted.quantile(
np.arange(global_histogram.shape[0]), global_histogram[:, i],
0.5)
ism_spec *= float(flux_range) / num_bins
ism_spec += flux_min
np.savez_compressed(output_file,
histogram=global_histogram,
ar_wavelength=ar_wavelength,
flux_range=[flux_min, flux_max],
ism_spec=ism_spec,
group_parameters=group_parameters)
def calc_fit_power_law(delta_f_snr_bins=snr_stats_total):
snr_bins = delta_f_snr_bins_helper.get_log_snr_axis()
y_quantile = np.zeros_like(snr_bins)
y1 = delta_f_snr_bins_helper.get_delta_f_axis()
for i in range(50):
y_quantile[i] = weighted.quantile(y1, delta_f_snr_bins[i], .9)
mask = [np.logical_and(-0 < snr_bins, snr_bins < 3)]
masked_snr_bins = snr_bins[mask]
# print("x2:", masked_snr_bins)
fit_params = lmfit.Parameters()
fit_params.add('a', -2., min=-5, max=-1)
fit_params.add('b', 1., min=0.1, max=20.)
fit_params.add('c', 0.08, min=0, max=0.2)
# make sure the exponent base is non-negative
fit_params.add('d', 3, min=-masked_snr_bins.min(), max=5)
fit_result = lmfit.minimize(fit_function,
fit_params,
kws={
'data': y_quantile[mask],
'x': masked_snr_bins
})
return fit_result, snr_bins, masked_snr_bins, y_quantile
def getQuantileIncome(quantile, state, race, sex, agegroup):
# NOTE: please see
# http://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMSDataDict14.pdf
# for column identifiers
# select all records matching the valid criteria
query = """
SELECT
PERNP, PWGTP
FROM
PUMS_2014_Persons
"""
query += getWhereClause(state, race, sex, agegroup)
cursor.execute(query)
# fetch all results and compute the quantile value
results = cursor.fetchall()
r = numpy.asarray(results, dtype=float)
if len(r) > 0:
pernp = r[:, 0]
pwgtp = r[:, 1]
return int(round(weighted.quantile(pernp, pwgtp, quantile),-2))
else:
return 0
def getQuantileIncome(quantile, state, race, sex, agegroup):
# NOTE: please see
# http://www2.census.gov/programs-surveys/acs/tech_docs/pums/data_dict/PUMSDataDict14.pdf
# for column identifiers
# select all records matching the valid criteria
query = """
SELECT
PERNP, PWGTP
FROM
PUMS_2014_Persons
"""
query += getWhereClause(state, race, sex, agegroup)
cursor.execute(query)
# fetch all results and compute the quantile value
results = cursor.fetchall()
r = numpy.asarray(results, dtype=float)
if len(r) > 0:
pernp = r[:, 0]
pwgtp = r[:, 1]
return int(round(weighted.quantile(pernp, pwgtp, quantile), -2))
else:
return 0
# simple attenuation to match observed data. #pdb.set_trace() att = [0.1,1.0] zratt = [0.2,1.8] att_gal = np.interp(GalArr['redshift'],zratt,att) loii = loii[0] + np.log10(att_gal) nFil = len(FiltArr['w']) CentralW = np.zeros(nFil) int_ab = np.zeros(nFil) p10p90W = np.zeros([nFil,2]) for i in range(nFil): CentralW[i] = wp.quantile(FiltArr['w'][i],FiltArr['t'][i],0.5) nz = np.where(FiltArr['w'][i] != 0) int_ab[i] = (integral.simps(FiltArr['t'][i][nz]/FiltArr['w'][i][nz],FiltArr['w'][i][nz])) p10p90W[i,0] = wp.quantile(FiltArr['w'][i][nz],FiltArr['t'][i][nz],0.1) p10p90W[i,1] = wp.quantile(FiltArr['w'][i][nz],FiltArr['t'][i][nz],0.9) print 'int_ab[i]',int_ab[i] mpc2cm_24 = 3.0856778 # mpc-to-cm /1e24 distance = np.sqrt(GalArr['pos'][:,0]**2 + GalArr['pos'][:,1]**2+GalArr['pos'][:,2]**2) log_dist = np.log10(distance) + np.log10(mpc2cm_24)+24.0 + np.log10(1+GalArr['redshift']) log_Fline = loii + 40.0 - (np.log10(4*np.pi)+2*(log_dist)) FLine = 10**log_Fline wgal = L_line*(1 + GalArr['redshift'])
# ax2.set_xlabel(r"${\rm SNR (red)}$", fontsize=14)
# ax2.set_ylabel(r"${\rm \left|\delta F\right|}$", fontsize=14)
# plt.set_cmap('gray')
# plt.plot(x, y)
result, snr_bins, masked_snr_bins, y_quantile = calc_fit_power_law()
print(power_law_to_string(result))
ax1.plot(snr_bins, y_quantile, linewidth=1.)
fit = fit_function(params=result.params, data=0, x=masked_snr_bins)
fit = np.clip(fit, 0, 1)
ax1.plot(masked_snr_bins, fit, color='deepskyblue',
linewidth=4.) # , linestyle=(0., (8., 8.)))
plt.xticks(np.arange(min(snr_bins), max(masked_snr_bins) + 1, 1.0))
fig.tight_layout()
print(
"test fit",
fit_function(params=result.params, data=0, x=[-1.5, -1., 0., 1., 2., 10.]))
plt.figure()
plt.plot(snr_bins, snr_stats_total.sum(axis=1))
total_snr_quantile_1 = weighted.quantile(snr_bins, snr_stats_total.sum(axis=1),
0.10)
total_snr_quantile_9 = weighted.quantile(snr_bins, snr_stats_total.sum(axis=1),
0.90)
print("SNR: 0.1 limit:", total_snr_quantile_1, "0.9 limit:",
total_snr_quantile_9)
plt.show()
def main(nproc, outfile):
"""
Creates files containing all emission line luminosities and
magnitudes for lightcone galaxies.
Calculation is split in a given number of processes, where
each computes the emission lines of a chunk of data independently
and outputs the results to a file.
Arguments:
nproc = number of processes
outfile = Output file name. (final name will be followed by a number)
"""
# nproc = argv[0]
# outfile = argv[1]
# if (len(argv) < 2):
# print 'you need to provide 2 arguments: nproc outfile.'
# print 'Exiting..'
# return 0
nproc = int(nproc)
IncludeMags = True
print 'nproc:', nproc, ' outfile:', outfile
sys.stdout.flush()
#L_line = 1215.67 # Angstroms
L_line = 3727.0 # Angstroms
#Lyaz,izr,GalArr,Vol = get_lya(zrange)
FiltArr, wrange = rf.read_filters()
jpaslims = jp.get_jpaslims()
app_type = 1 # 0 means per 3 arcsec diameter, 1 means per square arcsec
cosmo = FlatLambdaCDM(H0=73, Om0=0.25)
zrArr = comparat_zranges()
zminlist = map(float, zrArr['zmin'])
zmaxlist = map(float, zrArr['zmax'])
minz = np.min(zminlist)
maxz = np.max(zmaxlist)
#maxz = np.max(zrArr['zmax'])
print minz, maxz
minsfr = 0.1
nFil = len(FiltArr['w'])
CentralW = np.zeros(nFil)
int_ab = np.zeros(nFil)
p10p90W = np.zeros([nFil, 2])
for i in range(nFil):
CentralW[i] = wp.quantile(FiltArr['w'][i], FiltArr['t'][i], 0.5)
nz = np.where(FiltArr['w'][i] != 0)
int_ab[i] = (integral.simps(FiltArr['t'][i][nz] / FiltArr['w'][i][nz],
FiltArr['w'][i][nz]))
p10p90W[i, 0] = wp.quantile(FiltArr['w'][i][nz], FiltArr['t'][i][nz],
0.1)
p10p90W[i, 1] = wp.quantile(FiltArr['w'][i][nz], FiltArr['t'][i][nz],
0.9)
print 'int_ab[i]', int_ab[i]
print 'loading lightcone data'
props = ['redshift', 'pos', 'Mz', 'Mcold', 'sfr', 'vel', 'stellarmass']
print 'now loading photo-ionisation grid'
lineinfo, linesarr = read_photoion()
nline = lineinfo['nlines']
usezspace = 1
# magtype stores both the value of the magnitude (float) and the
# filter id (int) where that magnitude was computed.
magtype = np.dtype('f4, i4')
def read_lightcone_chunk(ip, nproc):
GalArr, lLyc, ngg, DistNz, zdist = readmock_chunk(ip,
nproc,
props_array=props,
zspace=usezspace,
sfrmin=500)
zcold = GalArr['Mcold'] / GalArr['Mz']
qpar = qZrelation(zcold) # assuming default pars.
# This stores all line luminosities for all galaxies
# LinesLumArr[i,j] = Luminosity of line j for galaxy i.
LinesLumArr = np.zeros((ngg, nline))
redshiftArr = np.zeros(ngg)
SfrArr = np.zeros(ngg)
MStellarArr = np.zeros(ngg)
ZArr = np.zeros(ngg)
print 'ip ' + str(
ip) + ', computing lines for ngals=', ngg, ' galaxies...'
sys.stdout.flush()
for ig in range(ngg):
LinesLumArr[ig, :] = integ_line(lineinfo,
linesarr,
qpar[ig],
zcold[ig],
lLyc[ig],
all_lines=True)
redshiftArr[ig] = GalArr['redshift'][ig]
SfrArr[ig] = GalArr['sfr'][ig]
MStellarArr[ig] = GalArr['stellarmass'][ig]
ZArr[ig] = zcold[ig]
# LinesLumArr[ig,il] = iline
print 'redshiftArr', redshiftArr[0:10]
print 'Lines[0,:]', LinesLumArr[0, :]
print 'Lines[10,:]', LinesLumArr[10, :]
print 'Lines[100,:]', LinesLumArr[100, :]
print 'zarr[100,:]', ZArr[0:100]
print 'Proc ip:' + str(ip) + ' done with that.'
# This stores the AB observed-frame apparent magnitude for all luminosities of all galaxies, indicating also to which filter
# the stored magnitudes corresponds to.
# MagsLumArr[i,j] = (AB magnitude, filter id) of line j of galaxy i.
MagsLumArr = np.zeros((ngg, nline), dtype=magtype)
realdist = np.sqrt(GalArr['pos'][:, 0]**2 + GalArr['pos'][:, 1]**2 +
GalArr['pos'][:, 2]**2)
log_dist = np.log10(realdist) + np.log10(mpc2cm_24) + 24.0 + np.log10(
1 + GalArr['redshift'])
if IncludeMags is True:
for il in range(nline):
L_line = lineinfo['lambda0'][il]
wgal = L_line * (1 + GalArr['redshift'])
log_Fline = LinesLumArr[:, il] - (np.log10(4 * np.pi) + 2 *
(log_dist))
FLine = 10**log_Fline
for i in range(ngg):
idFilter, Val = find_nearest_vector(CentralW, wgal[i])
if idFilter == 0:
if wgal[i] < p10p90W[0, 0]:
MagsLumArr[i, il] = (-99, -99)
continue
elif idFilter == nFil - 1:
if wgal[i] > p10p90W[nFil - 1, 1]:
MagsLumArr[i, il] = (99, 99)
continue
else:
nz = np.where(FiltArr['w'][idFilter] != 0)
lamb = FiltArr['w'][idFilter][nz]
tlamb = FiltArr['t'][idFilter][nz]
tlam0 = np.interp(wgal[i], lamb, tlamb)
# mag_app[i] = -2.5*(log_Fline[i]+ np.log10((wgal[i]*tlam0)/int_ab[idFilter]) - 18-np.log10(3.0)) - 48.60
mag_app = -2.5 * (log_Fline[i] + np.log10(
(wgal[i] * tlam0) / int_ab[idFilter]) - 18 -
np.log10(3.0)) - 48.60
MagsLumArr[i, il] = (mag_app, idFilter)
filename = outfile + '.' + str(ip)
nf = open(filename, "wb")
print 'writing file ' + filename
nf.write(struct.pack('l', ngg))
nf.write(struct.pack('i', nline))
LinesLumArr.tofile(nf)
if IncludeMags is True:
MagsLumArr.tofile(nf)
redshiftArr.tofile(nf)
SfrArr.tofile(nf)
MStellarArr.tofile(nf)
ZArr.tofile(nf)
nf.close()
print 'file ' + filename + ' written succesfully.'
return 1
if nproc == 1:
processes = read_lightcone_chunk(0, 1)
else:
print 'start processes'
processes = [
mp.Process(target=read_lightcone_chunk, args=(ip, nproc))
for ip in range(nproc)
]
for p in processes:
p.start()
for p in processes:
p.join()
return 1
def main(nproc,outfile):
"""
Creates files containing all emission line luminosities and
magnitudes for lightcone galaxies.
Calculation is split in a given number of processes, where
each computes the emission lines of a chunk of data independently
and outputs the results to a file.
Arguments:
nproc = number of processes
outfile = Output file name. (final name will be followed by a number)
"""
# nproc = argv[0]
# outfile = argv[1]
# if (len(argv) < 2):
# print 'you need to provide 2 arguments: nproc outfile.'
# print 'Exiting..'
# return 0
nproc = int(nproc)
IncludeMags = True
print 'nproc:',nproc,' outfile:',outfile
sys.stdout.flush()
#L_line = 1215.67 # Angstroms
L_line = 3727.0 # Angstroms
#Lyaz,izr,GalArr,Vol = get_lya(zrange)
FiltArr,wrange = rf.read_filters()
jpaslims = jp.get_jpaslims()
app_type = 1 # 0 means per 3 arcsec diameter, 1 means per square arcsec
cosmo = FlatLambdaCDM(H0=73, Om0=0.25)
zrArr = comparat_zranges()
zminlist = map(float,zrArr['zmin'])
zmaxlist = map(float,zrArr['zmax'])
minz = np.min(zminlist)
maxz = np.max(zmaxlist)
#maxz = np.max(zrArr['zmax'])
print minz,maxz
minsfr = 0.1
nFil = len(FiltArr['w'])
CentralW = np.zeros(nFil)
int_ab = np.zeros(nFil)
p10p90W = np.zeros([nFil,2])
for i in range(nFil):
CentralW[i] = wp.quantile(FiltArr['w'][i],FiltArr['t'][i],0.5)
nz = np.where(FiltArr['w'][i] != 0)
int_ab[i] = (integral.simps(FiltArr['t'][i][nz]/FiltArr['w'][i][nz],FiltArr['w'][i][nz]))
p10p90W[i,0] = wp.quantile(FiltArr['w'][i][nz],FiltArr['t'][i][nz],0.1)
p10p90W[i,1] = wp.quantile(FiltArr['w'][i][nz],FiltArr['t'][i][nz],0.9)
print 'int_ab[i]',int_ab[i]
print 'loading lightcone data'
props = ['redshift','pos','Mz','Mcold','sfr','vel','stellarmass']
print 'now loading photo-ionisation grid'
lineinfo,linesarr = read_photoion()
nline = lineinfo['nlines']
usezspace = 1
# magtype stores both the value of the magnitude (float) and the
# filter id (int) where that magnitude was computed.
magtype = np.dtype('f4, i4')
def read_lightcone_chunk(ip, nproc):
GalArr,lLyc,ngg,DistNz,zdist = readmock_chunk(ip,nproc,props_array = props,
zspace=usezspace,sfrmin=500)
zcold = GalArr['Mcold']/GalArr['Mz']
qpar = qZrelation(zcold) # assuming default pars.
# This stores all line luminosities for all galaxies
# LinesLumArr[i,j] = Luminosity of line j for galaxy i.
LinesLumArr = np.zeros((ngg,nline))
redshiftArr = np.zeros(ngg)
SfrArr = np.zeros(ngg)
MStellarArr = np.zeros(ngg)
ZArr = np.zeros(ngg)
print 'ip '+str(ip)+', computing lines for ngals=',ngg,' galaxies...'
sys.stdout.flush()
for ig in range(ngg):
LinesLumArr[ig,:] = integ_line(lineinfo,linesarr,qpar[ig],zcold[ig],lLyc[ig],all_lines=True)
redshiftArr[ig] = GalArr['redshift'][ig]
SfrArr[ig] = GalArr['sfr'][ig]
MStellarArr[ig] = GalArr['stellarmass'][ig]
ZArr[ig] = zcold[ig]
# LinesLumArr[ig,il] = iline
print 'redshiftArr' , redshiftArr[0:10]
print 'Lines[0,:]' , LinesLumArr[0,:]
print 'Lines[10,:]' , LinesLumArr[10,:]
print 'Lines[100,:]' , LinesLumArr[100,:]
print 'zarr[100,:]' , ZArr[0:100]
print 'Proc ip:'+str(ip)+' done with that.'
# This stores the AB observed-frame apparent magnitude for all luminosities of all galaxies, indicating also to which filter
# the stored magnitudes corresponds to.
# MagsLumArr[i,j] = (AB magnitude, filter id) of line j of galaxy i.
MagsLumArr = np.zeros((ngg,nline),dtype=magtype)
realdist = np.sqrt(GalArr['pos'][:,0]**2 + GalArr['pos'][:,1]**2+GalArr['pos'][:,2]**2)
log_dist = np.log10(realdist) + np.log10(mpc2cm_24)+24.0 + np.log10(1+GalArr['redshift'])
if IncludeMags is True:
for il in range(nline):
L_line = lineinfo['lambda0'][il]
wgal = L_line*(1 + GalArr['redshift'])
log_Fline = LinesLumArr[:,il] - (np.log10(4*np.pi)+2*(log_dist))
FLine = 10**log_Fline
for i in range(ngg):
idFilter,Val = find_nearest_vector(CentralW,wgal[i])
if idFilter == 0:
if wgal[i] < p10p90W[0,0]:
MagsLumArr[i,il] = (-99,-99)
continue
elif idFilter == nFil-1:
if wgal[i] > p10p90W[nFil-1,1]:
MagsLumArr[i,il] = (99,99)
continue
else:
nz = np.where(FiltArr['w'][idFilter] != 0)
lamb = FiltArr['w'][idFilter][nz]
tlamb = FiltArr['t'][idFilter][nz]
tlam0 = np.interp(wgal[i],lamb,tlamb)
# mag_app[i] = -2.5*(log_Fline[i]+ np.log10((wgal[i]*tlam0)/int_ab[idFilter]) - 18-np.log10(3.0)) - 48.60
mag_app = -2.5*(log_Fline[i]+ np.log10((wgal[i]*tlam0)/int_ab[idFilter]) - 18-np.log10(3.0)) - 48.60
MagsLumArr[i,il] = (mag_app, idFilter)
filename = outfile + '.' + str(ip)
nf = open(filename,"wb")
print 'writing file '+filename
nf.write(struct.pack('l',ngg))
nf.write(struct.pack('i',nline))
LinesLumArr.tofile(nf)
if IncludeMags is True:
MagsLumArr.tofile(nf)
redshiftArr.tofile(nf)
SfrArr.tofile(nf)
MStellarArr.tofile(nf)
ZArr.tofile(nf)
nf.close()
print 'file '+filename+' written succesfully.'
return 1
if nproc == 1:
processes = read_lightcone_chunk(0,1)
else:
print 'start processes'
processes = [mp.Process(target=read_lightcone_chunk, args=(ip,nproc)) for ip in range(nproc)]
for p in processes:
p.start()
for p in processes:
p.join()
return 1