def modified_zscore(col):
"""Makes calulations for Modified Z-Score"""
col = col.dropna()
med_col = col.median()
med_abs_dev = MAD(col)
mod_z = 0.6745 * ((col - med_col) / med_abs_dev)
return np.abs(mod_z)
def statResid(df, colResid):
if colResid == None:
statDict = {'Num objects': nan, 'Std': nan, 'NMAD': nan, 'Mean': nan}
else:
statDict = {
'Num objects': int(len(df)),
'Std': around(std(df[colResid]), 3),
'NMAD': around(MAD(df[colResid]), 3),
'Mean': around(mean(df[colResid]), 4)
}
if len(df) > 0:
statDict['% outl_15'] = around(
len(df[abs(df[colResid]) > 0.15]) / len(df) * 100, 2)
else:
statDict['% outl_15'] = 0
return statDict
AVG_MEASURE,
color=colors[index])
ax[0].bar(index, np.nanmean(AVG_MEASURE), alpha=0.5, color=colors[index])
ax[0].errorbar(index,
np.nanmean(AVG_MEASURE),
yerr=SEM(AVG_MEASURE),
color=colors[index])
avg.append(AVG_MEASURE)
print('Avg Amplitude per mice at {} = {} +/- {}'.format(
group, round(np.nanmean(AVG_MEASURE), RF),
round(np.nanstd(AVG_MEASURE), RF)))
print('Med Amplitude per mice at {} = {} +/- {}'.format(
group, round(np.nanmedian(AVG_MEASURE), RF),
round(MAD(AVG_MEASURE), RF)))
ax[1].scatter(np.ones(len(TOTAL_MEASURE)) * index,
TOTAL_MEASURE,
color=colors[index])
ax[1].bar(index, np.nanmean(TOTAL_MEASURE), alpha=0.5, color=colors[index])
ax[1].errorbar(index,
np.nanmean(TOTAL_MEASURE),
yerr=SEM(TOTAL_MEASURE),
color=colors[index])
total.append(TOTAL_MEASURE)
print('Total (MEAN) currents per mice at {} = {} +/- {}'.format(
group, round(np.nanmean(TOTAL_MEASURE), RF),
round(np.nanstd(TOTAL_MEASURE), RF)))
globalDf.append(activationDfcopy)
#Intra group statistics
print('---- Intra Group Statistics ----')
print('Avg activation (%) in {}: {}+/-{}'.format(
deepValues[0], round(activationDf[deepValues[0]].mean(), RF),
round(activationDf[deepValues[0]].std(), RF)))
print('Avg activation (%) in {}: {}+/-{}'.format(
deepValues[1], round(activationDf[deepValues[1]].mean(), RF),
round(activationDf[deepValues[1]].std(), RF)))
print('Med activation (%) in {}: {}+/-{}'.format(
deepValues[0], round(activationDf[deepValues[0]].median(), RF),
round(MAD(activationDf[deepValues[0]].values), RF)))
print('Med activation (%) in {}: {}+/-{}'.format(
deepValues[1], round(activationDf[deepValues[1]].median(), RF),
round(MAD(activationDf[deepValues[1]].values), RF)))
kwTest = stats.kruskal(activationDf[deepValues[0]],
activationDf[deepValues[1]])
print('Kruskal Wallis test H value = {}'.format(kwTest[0]))
print('Kruskal Wallis test p value = {}'.format(kwTest[1]))
if kwTest[1] <= 0.05:
print('Post Hoc MWU test')
#Extract significant measures on the whole map
sitesToKeep = [
x for (x, y) in zip(measures, zscores) if y >= zscoreCut
]
for i in sitesToKeep:
currents.append(i)
#Average values
distributions.append(currents)
avg = np.nanmean(currents)
std = np.nanstd(currents)
med = np.nanmedian(currents)
mad = MAD(currents)
print('MEAN {} -> {} +/- {} pA'.format(group, round(avg, RF),
round(std, RF)))
print('MEDIAN {} -> {} +/- {} pA'.format(group, round(med, RF),
round(mad, RF)))
print('Min. current: {} pA'.format(min(currents)))
print('Max. current: {} pA'.format(max(currents)))
sn.distplot(currents, kde=True, color=colors[index], ax=ax)
sn.distplot(currents,
hist=True,
kde=True,
color=colors[index],
ax=curves[index])
PropDf = pd.DataFrame()
PropDf['Local'] = LocalCount
PropDf['Distal'] = DistCount
#Append in global dataframes
LOCAL_AMPS.append(pd.DataFrame(individualLocal))
DISTAL_AMPS.append(pd.DataFrame(individualDist))
LOCAL_PROP.append(pd.DataFrame(LocalCount))
DISTAL_PROP.append(pd.DataFrame(DistCount))
avgLocal = np.nanmean(individualLocal)
stdLocal = np.nanstd(individualLocal)
semLocal = SEM(individualLocal)
medLocal = np.nanmedian(individualLocal)
madLocal = MAD([x for x in individualLocal if math.isnan(x) == False])
avgDist = np.nanmean(individualDist)
stdDist = np.nanstd(individualDist)
semDist = SEM(individualDist)
medDist = np.nanmedian(individualDist)
madDist = MAD([x for x in individualDist if math.isnan(x) == False])
countLocalavg = np.nanmean(LocalCount)
countLocalstd = np.nanstd(LocalCount)
countLocalSEM = SEM(LocalCount)
countLocalMed = np.nanmedian(LocalCount)
countLocalMad = MAD([x for x in LocalCount if math.isnan(x) == False])
#Append in list
AVG_MEASURE.append(avgMeasure)
TOTAL_MEASURE.append(totalMeasure)
AVG_PROP.append(proportionActiveSite)
# if group == 'LS':
# box[0].annotate(manip,(index,avgMeasure))
# box[1].annotate(manip,(index,totalMeasure))
# box[2].annotate(manip,(index,proportionActiveSite))
print ('Avg Amp+/-SD (map wise): {}+/-{}'.format(round(np.nanmean(AVG_MEASURE),2), round(np.nanstd(AVG_MEASURE),2)))
print ('Avg activation (%)+/-SD: {}+/-{}'.format(round(np.nanmean(AVG_PROP),2), round(np.nanstd(AVG_PROP),2)))
print ('Med activation (%)+/-SD: {}+/-{}'.format(round(np.nanmedian(AVG_PROP),2), round(MAD(AVG_PROP),2)))
#Scatter plot
ax[0].scatter(np.ones(len(AVG_MEASURE))*index,AVG_MEASURE,color=colors[index])
ax[0].bar(index,np.nanmean(AVG_MEASURE),alpha=0.5,color=colors[index])
ax[0].errorbar(index,np.nanmean(AVG_MEASURE),yerr=SEM(AVG_MEASURE),color=colors[index])
avg.append(AVG_MEASURE)
ax[1].scatter(np.ones(len(TOTAL_MEASURE))*index,TOTAL_MEASURE,color=colors[index])
ax[1].bar(index,np.nanmean(TOTAL_MEASURE),alpha=0.5,color=colors[index])
ax[1].errorbar(index,np.nanmean(TOTAL_MEASURE),yerr=SEM(TOTAL_MEASURE),color=colors[index])
total.append(TOTAL_MEASURE)
def makePlot(data_all, probs_mean, fname, Plx_offset=0.029):
"""
Make plots using the final probabilities in the "_probs.dat" files.
"""
# prob_cut = min(prob_cut, probs_mean.max())
msk_memb = probs_mean >= min(prob_cut, probs_mean.max())
fig = plt.figure(figsize=(15, 10))
G = gridspec.GridSpec(4, 3)
ax1 = plt.subplot(G[0:2, 0])
ax21 = plt.subplot(G[0:1, 1])
ax22 = plt.subplot(G[1:2, 1])
ax3 = plt.subplot(G[0:2, 2])
ax4 = plt.subplot(G[2:4, 0])
ax5 = plt.subplot(G[2:4, 1])
ax6 = plt.subplot(G[2:4, 2])
plt.suptitle(
"Percentile distance used to estimate the 'eps': {}".format(perc_cut))
data_all[col_names['plx']] += Plx_offset
# # Reachability plot
# space = np.arange(len(data))
# msk = reachability < eps
# ax1.plot(space[~msk], reachability[~msk], 'k.', alpha=0.3)
# ax1.plot(space[msk], reachability[msk], 'g.')
# ax1.axhline(eps, c='k', ls='-.', label="eps={:.3f}".format(eps))
# ax1.set_ylabel('Reachability (epsilon distance)')
# ax1.set_title('Reachability Plot (min_samples={})'.format(
# min_samples))
# ymin, ymax = max(0, eps - eps * .5), min(3. * eps, max(reachability))
# ax1.set_xlim(0, max(space[reachability < ymax]))
# ax1.set_ylim(ymin, ymax)
# ax1.legend()
ax1.hist(probs_mean, alpha=.5)
ax1.axvline(prob_cut, c='g', zorder=6)
ax12 = ax1.twinx()
ax12.yaxis.set_minor_locator(AutoMinorLocator(4))
xx, yy = np.linspace(probs_mean.min(), probs_mean.max(), 100), []
for pr in xx:
yy.append((probs_mean >= pr).sum())
ax12.plot(xx,
yy,
c='r',
lw=3,
label=r"$P_{{cut}}$={:.2f}".format(prob_cut))
ax12.set_ylabel(r'$N_{stars}$')
# ax12.set_yscale('log')
# ax12.set_yticks([])
ax1.set_xlabel('Probabilities')
ax1.set_ylabel(r'$\log (N)$')
ax1.set_yscale('log')
plt.legend()
plx_prob, pmRA_prob, pmDE_prob = [], [], []
for pp in np.arange(.05, .95, .01):
msk = probs_mean >= pp
plx_prob.append([
pp, data_all[col_names['plx']][msk].mean(),
data_all[col_names['plx']][msk].std()
])
pmRA_prob.append([
pp, data_all[col_names['pmra']][msk].mean(),
data_all[col_names['pmra']][msk].std()
])
pmDE_prob.append([
pp, data_all[col_names['pmde']][msk].mean(),
data_all[col_names['pmde']][msk].std()
])
plx_prob, pmRA_prob, pmDE_prob = [
np.array(_).T for _ in (plx_prob, pmRA_prob, pmDE_prob)
]
ax21.plot(pmDE_prob[0], pmDE_prob[1], c='b', label='pmDE')
ax21.fill_between(pmDE_prob[0],
pmDE_prob[1] - pmDE_prob[2],
pmDE_prob[1] + pmDE_prob[2],
color='blue',
alpha=0.1)
ax21.axvline(prob_cut, c='g', zorder=6)
ax21.set_xlabel(r'$P_{cut}$')
ax21.set_ylabel(r'$\mu_{\delta}$ [mas/yr]')
ax22.plot(pmRA_prob[0], pmRA_prob[1], c='r', label='pmRA')
ax22.fill_between(pmRA_prob[0],
pmRA_prob[1] - pmRA_prob[2],
pmRA_prob[1] + pmRA_prob[2],
color='red',
alpha=0.1)
ax22.axvline(prob_cut, c='g', zorder=6)
ax22.set_ylabel(r'$\mu_{\alpha} \cos \delta$ [mas/yr]')
ax22.set_xlabel(r'$P_{cut}$')
ax3.plot(plx_prob[0], plx_prob[1])
ax3.fill_between(plx_prob[0],
plx_prob[1] - plx_prob[2],
plx_prob[1] + plx_prob[2],
color='k',
alpha=0.1)
ax3.axvline(prob_cut, c='g', zorder=6)
ax3.set_ylabel(r'$Plx$')
ax3.set_xlabel(r'$P_{cut}$')
ax4.set_title("N={}".format(len(data_all[col_names['ra']][msk_memb])))
ax4.scatter(data_all[col_names['ra']][msk_memb],
data_all[col_names['dec']][msk_memb],
marker='o',
edgecolor='w',
lw=.3,
zorder=5)
ax4.plot(data_all[col_names['ra']][~msk_memb],
data_all[col_names['dec']][~msk_memb],
'k.',
alpha=0.3,
zorder=1)
ax4.set_xlabel('RA')
ax4.set_ylabel('DE')
ax4.set_xlim(max(data_all[col_names['ra']]),
min(data_all[col_names['ra']]))
ax4.set_ylim(min(data_all[col_names['dec']]),
max(data_all[col_names['dec']]))
# PMs plot
pmRA_mean = (
data_all[col_names['pmra']][msk_memb] /
np.cos(np.deg2rad(data_all[col_names['pmde']][msk_memb]))).mean()
pmDE_mean = data_all[col_names['pmde']][msk_memb].mean()
ax5.set_title(r"$(\mu_{\alpha}, \mu_{\delta})=$" +
"({:.3f}, {:.3f})".format(pmRA_mean, pmDE_mean))
ax5.scatter(data_all[col_names['pmra']][msk_memb],
data_all[col_names['pmde']][msk_memb],
marker='.',
edgecolor='w',
lw=.1,
alpha=.7,
zorder=5)
ax5.plot(data_all[col_names['pmra']][~msk_memb],
data_all[col_names['pmde']][~msk_memb],
'k+',
alpha=0.3,
zorder=1)
ax5.set_xlabel(r'$\mu_{\alpha} \cos \delta$ [mas/yr]')
ax5.set_ylabel(r'$\mu_{\delta}$ [mas/yr]')
xmin, xmax = np.percentile(data_all[col_names['pmra']][~msk_memb],
(25, 75))
ymin, ymax = np.percentile(data_all[col_names['pmde']][~msk_memb],
(25, 75))
pmra_MAD = MAD(data_all[col_names['pmra']][msk_memb], scale='normal')
pmde_MAD = MAD(data_all[col_names['pmde']][msk_memb], scale='normal')
xclmin = data_all[col_names['pmra']][msk_memb].mean() - 5. + pmra_MAD
xclmax = data_all[col_names['pmra']][msk_memb].mean() + 5. + pmra_MAD
yclmin = data_all[col_names['pmde']][msk_memb].mean() - 5. + pmde_MAD
yclmax = data_all[col_names['pmde']][msk_memb].mean() + 5. + pmde_MAD
ax5.set_xlim(max(xmax, xclmax), min(xmin, xclmin))
ax5.set_ylim(min(ymin, yclmin), max(ymax, yclmax))
# Plxs plot
ax6.set_title("Plx offset: +{}".format(Plx_offset))
xmin = np.percentile(data_all[col_names['plx']][msk_memb], 1) -\
3. * data_all[col_names['plx']][msk_memb].std()
xmax = np.percentile(data_all[col_names['plx']][msk_memb], 95) +\
3. * data_all[col_names['plx']][msk_memb].std()
msk1 = np.logical_and.reduce([(data_all[col_names['plx']] > -2),
(data_all[col_names['plx']] < 4),
(msk_memb)])
msk2 = np.logical_and.reduce([(data_all[col_names['plx']] > -2),
(data_all[col_names['plx']] < 4),
(~msk_memb)])
ax6.hist(data_all[col_names['plx']][msk1],
20,
density=True,
alpha=.7,
zorder=5)
ax6.hist(data_all[col_names['plx']][msk2],
75,
color='k',
alpha=0.3,
density=True,
zorder=1)
plx_mean = data_all[col_names['plx']][msk_memb].mean()
plx_16, plx_84 = np.percentile(data_all[col_names['plx']][msk_memb],
(16, 84))
ax6.axvline(plx_mean,
c='r',
label=r"$Plx_{{mean}}={:.3f}_{{{:.3f}}}^{{{:.3f}}}$".format(
plx_mean, plx_16, plx_84),
zorder=6)
ax6.axvline(plx_16, c='orange', ls='--', zorder=6)
ax6.axvline(plx_84, c='orange', ls='--', zorder=6)
ax6.set_xlabel('Plx')
ax6.set_xlim(xmin, xmax)
ax6.legend()
file_out = 'output/' + fname + '.png'
fig.tight_layout()
plt.savefig(file_out, dpi=150, bbox_inches='tight')
plt.close()