def make_entry(match,SI,intercept,SI_err,intercept_err,g_stats,accept_type,low_resids,chired): '''Gather all of the information together in to a format that can easily be read in to create the output VOTable''' source = mkl.get_srcg(match) source.SI = SI source.intercept = intercept source.SI_err = SI_err source.intercept_err = intercept_err source.low_resids = low_resids sources.append(source) g_stats.accept_type = accept_type sources_stats.append(g_stats) source.chi_resid = chired
def make_entry(match, SI, intercept, SI_err, intercept_err, g_stats,
accept_type, low_resids, chired, match_type):
'''Gather all of the information together in to a format that can easily be read in to create
the output FITS or VOTable'''
source = mkl.get_srcg(match)
source.SI = SI
source.intercept = intercept
source.SI_err = SI_err
source.intercept_err = intercept_err
source.low_resids = low_resids
sources.append(source)
g_stats.accept_type = accept_type
g_stats.match_type = match_type
sources_stats.append(g_stats)
source.chi_resid = chired
for cat, name in zip(source.cats[1:], source.names[1:]):
source.matched_names.append((cat, name))
def single_match_test(src_all, comp, accepted_matches, accepted_inds, g_stats,
num_matches, repeated_cats, matches):
'''Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up'''
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([match])
##Gather source information
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob) + 1
match_crit = "Combination (%d)\npossible\n%s repeated cats" % (
dom_num, repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr = np.pi / 180.0
delta_RA = np.arccos(
(np.cos(closeness * dr) - np.sin(src_all.decs[0] * dr)**2) /
np.cos(src_all.decs[0] * dr)**2) / dr
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra != -100000.0]
dec_test = [dec for dec in src_g.decs if dec != -100000.0]
small_test = []
for ra, dec in zip(ras_test, dec_test):
##We set up delta_RA to give us the equivalent offset in RA that corresponds to the
##resolution, so we use the offset in RA, not the arcdistance
prim_ra = src_all.ras[0]
ra_dist = ra - prim_ra
##Code to cope if one source 359.9, other 0.1 etc.
if abs(ra_dist) > 180.0:
if ra > 180.0:
ra -= 360.0
ra_dist = prim_ra - ra
else:
prim_ra -= 360.0
ra_dist = ra - prim_ra
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist / ra_axis)**2 + (dec_dist / dec_axis)**2
if ell_test <= 1:
small_test.append('yes')
else:
##Otherwise, fails
small_test.append('no')
#no_names.append(repeat_name) #Note the name of the sources that are far away
#Fail the positional test if a source is outside of the resolution plus position error
close_test = 'passed'
if 'no' in small_test: close_test = 'failed'
##If prob is higher than threshold, ignore position of sources and accept the match
if prob > high_prob:
##Accept the source, put it in a way that can be read when constructing the final table
if chi_resids[0] <= 2:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'position', 0, chi_resids[0],
'isolated')
else:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'position', 1, chi_resids[0],
'isolated')
make_accept(comp, g_stats, 'position', accepted_inds)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test == 'passed' or prob > low_prob:
##IF below eith threshold, append with the applicable fit label
if jstat_resids[0] <= jstat_thresh or chi_resids[0] <= chi_thresh:
if chi_resids[0] <= 2:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'spectral', 0,
chi_resids[0], 'isolated')
else:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'spectral', 1,
chi_resids[0], 'isolated')
make_accept(comp, g_stats, 'spectral', accepted_inds)
else:
g_stats.retained_matches = 1
##Put accepted inds as [0] just to have something outputted to the investigate text file -
##accepted_inds is empty is rejecting at this stage
make_rejection(comp, g_stats, 'spectral', [0])
else:
g_stats.retained_matches = 1
make_rejection(comp, g_stats, 'position', [0])
##If just one combo positionally possible, do a single combo check
elif len(accepted_matches) == 1:
single_match_test(src_all, comp, accepted_matches, accepted_inds,
g_stats, len(matches), repeated_cats, matches)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != '-100000.0']))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs,
num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != 'none':
jstat_resids, params, bses, chi_resids = mkl.calculate_resids(
[accepted_matches[dom_source]])
if chi_resids[0] <= 2:
make_entry(accepted_matches[dom_source], params[0][0],
params[0][1], bses[0][0], bses[0][1], g_stats,
'spectral', 0, chi_resids[0], 'dominant')
else:
make_entry(accepted_matches[dom_source], params[0][0],
params[0][1], bses[0][0], bses[0][1], g_stats,
'spectral', 1, chi_resids[0], 'dominant')
make_accept(comp, g_stats, 'spectral', accepted_inds)
##If nothing dominates, send to check if a combined source works
else:
def create_plot(comp,accepted_inds,match_crit,dom_crit,outcome):
'''The main plotting function that takes the relevant data and plots the outcome'''
###Split the information up as needed
chunks = comp.split('START_COMP')
all_info = chunks[0].split('\n')
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry=='': del all_info[all_info.index(entry)]
for entry in all_info:
if 'START' in entry: del all_info[all_info.index(entry)]
matches = chunks[1].split('\n')
del matches[0],matches[-2:]
##See how many matches there are, and set up the number of plots needed. If there are
##more than 16 matches, only plot the top 15 and print how many more matches there
##were - saves heaps of tiny little plots from appearing
num_matches = len(matches)
skip_16 = 'no'
if num_matches==1:
width = 1
height = 2
elif num_matches>16:
width = 4
height = 4
skip_16 = 'yes'
else:
width = int(num_matches**0.5)
height = num_matches/width
if num_matches%width==0:
pass
else:
height+=1
##Sets up a grid layout for the whole of the figure. We'll use half later on for
##the individual plots
gs = gridspec.GridSpec(height,2*width)
##Need a big plot!
fig = plt.figure(figsize = (18,11))
##Find out the ra,dec of the base catalogue source
info=all_info[0].split()
ra_main = float(info[2])
dec_main = float(info[4])
##Set up dedicated left plots
main_dims = [0.16, 0.5, 0.29, 0.35]
spec_dims = [0.16, 0.1, 0.29, 0.35]
ax_main,ax_spectral,tr_fk5,wcs = make_left_plots(fig,main_dims,spec_dims,ra_main,dec_main)
##Find the limits out to search area - have to do each edge individual,
##because of the arcdistance projection malarky
##Even at the same dec, 3 arcmins apart in RA doesn't translate to 3arcmin arcdist - projection
##fun. Can use law of cosines on a sphere to work out appropriate delta RA. Use this to define plot
##limits for a nice looking plot
delta_RA = np.arccos((np.cos((2*closeness)*dr)-np.sin(dec_main*dr)**2)/np.cos(dec_main*dr)**2)/dr
plot_lim = (2*closeness) + (0.1/60.0)
ra_up_lim = ra_main + delta_RA + (0.1/60.0)
ra_down_lim = ra_main - delta_RA - (0.1/60.0)
dec_up_lim = dec_main + plot_lim
dec_down_lim = dec_main - plot_lim
##Plot the individual combination plots - do this first so the error bars go over
##the top of the line plots
spec_labels = []
SIs = []
for i in xrange(height):
for j in range(width,2*width):
if i*j == 21:
if skip_16=='yes':
ax = plt.subplot(gs[i,j])
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.text(0.5,0.5,"And %d\nother\nplots" %(num_matches-15),transform=ax.transAxes,verticalalignment='center',horizontalalignment='center',fontsize=16)
else:
try:
ind = (i*width)+(j-width)
match = matches[ind]
ax = plt.subplot(gs[i,j])
ax.set_xticklabels([])
ax.set_yticklabels([])
##TODO - if plot centred on or close to RA,Dec = 0,0 then going to get wrapping problems. Should be able to pull the need
##for a wrap from ra_down_lim,ra_up_lim - one should be <0.0, or >360.0. Need to happen inside plot_ind
prob,resid,spec_plot,params = plot_ind(match,ax,ind,ax_spectral,ra_down_lim,ra_up_lim,dec_down_lim,dec_up_lim,dom_crit,outcome)
if spec_plot=='na':
pass
else:
SIs.append([params[0],str(ind+1)])
spec_labels.append(spec_plot)
except IndexError:
pass
#===========================================================#
##Plot the matching criteria information
match1 = matches[0].split()
src_g = mkl.get_srcg(match1)
text_axes = fig.add_axes([0.45,0.5,0.125,0.35])
text_axes.axis('off')
##Plot the matching information
props = dict(boxstyle='round', facecolor='w',lw='1.5')
text_axes.text(0.5,0.5,'Match Criteria:\n%s\n\nDominace Test:\n%s\n\nOutcome:\n%s' %(match_crit,dom_crit,outcome),
bbox=props,transform=text_axes.transAxes,verticalalignment='center',horizontalalignment='center',fontsize=16)
all_fluxs = []
##Fill the left hand plots with information goodness
all_freqs = fill_left_plots(all_info,ra_main,dec_main,ax_main,ax_spectral,tr_fk5,wcs,all_fluxs,ra_down_lim,ra_up_lim,dec_down_lim,dec_up_lim,delta_RA)
##If no repeated catalogues to combine, skip
if num_matches==0 or num_matches==1:
pass
##Otherwise, plot the combined fluxes
else:
##Calculate and plot the combined fluxes of the two sources, even if source one or two has been accepted
##just as a guide
src_all = mkl.get_allinfo(all_info)
##If positionally impossible don't plot combined info
if accepted_inds=='Nope':
pass
##If only one position possible, don't plot combined info
elif len(accepted_inds) == 1:
pass
##Otherwise, see what the combined fluxes look like
else:
comb_crit, ra_ws, rerr_ws, dec_ws, derr_ws, temp_freqs, comb_freqs, comb_fluxs, comb_ferrs, comb_fit, comb_jstat, comb_chi_red, combined_names, set_freqs, set_fluxs, set_fits = mkl.combine_flux(src_all,src_g,accepted_inds,'plot=yes',len(matches))
##If the criteria sent the double to be combined, actually plot the fitted line
if dom_crit == 'No dom. source':
for freq,flux in zip(set_freqs,set_fluxs):
ax_spectral.plot(freq,flux,linestyle='--',linewidth=1,color='r')
split_colors = ['#AE70ED','#FFB60B','#62A9FF','#59DF00']
for fit in set_fits:
ind = set_fits.index(fit)
ax_spectral.plot(set_freqs[ind],set_fluxs[ind],linestyle='--',linewidth=1.0,color=split_colors[ind],alpha=0.7)
split_p, = ax_spectral.plot(temp_freqs,np.exp(fit.params[1] + np.log(temp_freqs)*fit.params[0]),linestyle='-',linewidth=1.5,color=split_colors[ind])
spec_labels.append(split_p)
SIs.append([fit.params[0],'split %d' %(ind+1)])
bright_colours = ['#FF6600','#33FF33','#FF47A3','#00ebb3']
for freq in xrange(len(comb_freqs)):
plot_errors_comb('*',bright_colours[freq],comb_freqs[freq],comb_fluxs[freq],comb_ferrs[freq],'combo',16,ax_spectral)
comb_p, = ax_spectral.plot(temp_freqs,np.exp(comb_fit.fittedvalues),linestyle='--',linewidth=1.5,color='k')
spec_labels.append(comb_p)
SIs.append([comb_fit.params[0],'comb'])
##Send the combined fluxes to the all_fluxs so that ax_spectral is scaled appropriately
for flux in comb_fluxs:
all_fluxs.append(flux)
for pos in xrange(len(ra_ws)):
patch = plot_pos_comb('*',bright_colours[pos],ra_ws[pos],dec_ws[pos],rerr_ws[pos],derr_ws[pos],combined_names[pos],16,ax_main,ax_main.get_transform("fk5"))
scale_spectral(all_fluxs,all_freqs,ax_spectral)
##==============================================================
fig.tight_layout()
fig.subplots_adjust(bottom=0.1)
fig.subplots_adjust(left=0.15)
##Make room at the top of the plot for a legend for ax_main, make the legend
fig.subplots_adjust(top=0.85)
leg_labels = [r'$\alpha_{%s}$ = %.2f' %(SI[1],SI[0]) for SI in SIs]
main_handles,main_labels = ax_main.get_legend_handles_labels()
main_leg = fig.add_axes([0.05,0.87,0.9,0.05])
main_leg.axis('off')
main_leg.legend(main_handles,main_labels,loc='lower center',prop={'size':12},ncol=8) #,bbox_to_anchor=(0,1.02),
spec_leg = fig.add_axes([0.45,0.1,0.125,0.35])
spec_leg.axis('off')
##Stop the legend from having so many entries that it goes off the plot
if len(spec_labels)>11:
trim_labels = spec_labels[:10]
trim_labels.append(spec_labels[-1])
trim_legs = leg_labels[:10]
trim_legs.append(leg_labels[-1])
spec_leg.legend(trim_labels,trim_legs,loc='center',prop={'size':14},fancybox=True)
else:
spec_leg.legend(spec_labels,leg_labels,loc='center',prop={'size':14},fancybox=True)
##Create an axes to contain patches for an ellipse legend
patch_leg = fig.add_axes([0.015,0.1,0.06,0.75])
patch_leg.set_xticks([])
patch_leg.set_yticks([])
patch_leg.set_xticklabels([])
patch_leg.set_yticklabels([])
##See what catalogues are present in the match
present_cats = [cat for cat in set(src_g.cats) if cat!='-100000.0']
##Scale accordingly
increment = 1.0/(2+len(present_cats))
ell_positions = np.arange(increment/2,1,increment)
##Find the axes coord transform
patch_trans = patch_leg.transAxes
##Plot and name the resolution ellipse
ell = patches.Ellipse((0.5,ell_positions[-2]),0.9,increment-0.05,angle=0,
transform=patch_trans, linestyle='dashed',fc='none',lw=1.1,color='gray')
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[-2],'Resolution\n+ error',
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=14)
##Plot and name the search ellipse
ell = patches.Ellipse((0.5,ell_positions[-1]),0.9,increment-0.05,angle=0,
transform=patch_trans, linestyle='dashdot',fc='none',lw=1.1,color='k')
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[-1],'Search\nradius',
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=14)
##Use the same method as plot_all - for some reason was getting transform errors.
##so do it separately here (sigh)
for cat in present_cats:
col_ind = matched_cats.index(cat)
position_ind = present_cats.index(cat)
patch_leg.errorbar(0.5,ell_positions[position_ind],0.01,0.075,marker=markers[col_ind],ms=8,mfc=marker_colours[col_ind],
mec=marker_colours[col_ind],ecolor=marker_colours[col_ind],markeredgewidth=1,label='meh',linestyle='None',transform=patch_trans)
ell = patches.Ellipse((0.5,ell_positions[position_ind]),0.9,increment-0.05,angle=0, transform=patch_trans,
fc=ell_colours1[col_ind],color=ell_colours2[col_ind],alpha=alphas[col_ind])
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[position_ind]-(increment/2-0.04),cat,
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=16)
if save_plots:
plt.savefig('%s-pumaplot.png' %all_info[0].split()[1],bbox_inches='tight',dpi=100)
plt.close()
else:
plt.show()
def make_plots(comp, i):
##Get the information into nice usable forms, and get rid of empty/pointless
##entries
chunks = comp.split("START_COMP")
all_info = chunks[0].split("\n")
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry == "":
del all_info[all_info.index(entry)]
for entry in all_info:
if "START" in entry:
del all_info[all_info.index(entry)]
matches = chunks[1].split("\n")
del matches[0], matches[-2:]
##Put the information for every source in the matched group in one source_group() class
##(see apply_criteria_lib for source_group())
src_all = mkl.get_allinfo(all_info)
# print src_all.names[0]
##This line applies positional criteria, and tells us if a simple one catalogue repeat source, returning
##how many combinations are possible and statistics on them
repeated_cats, accepted_matches, accepted_inds, accepted_probs, jstats, chi_reds, g_stats = mkl.matches_retained(
src_all, matches
)
match_crit = "%d of %d \ncombinations \npossible \n%s repeated cats" % (
len(accepted_matches),
len(matches),
repeated_cats,
)
##If no combinations are possible, reject all info (goes into the eyeball document)
if len(accepted_matches) == 0:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Positionally\nimpossible",
"N/A",
len(matches),
plot_reject,
src_all,
"position",
i,
)
##If just one combo positionally possible, do a single combo check
elif len(accepted_matches) == 1:
i = single_match_test(
src_all, comp, accepted_matches, accepted_inds, g_stats, len(matches), repeated_cats, matches, i
)
##(Any plotting gets done within single_match_test)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != "-100000.0"]))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs, num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != "none":
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([accepted_matches[dom_source]])
##Find the probs of all the matches, and use the prob of the dom match to see what number match was accepted
all_probs = [float(match.split()[-1]) for match in matches]
accepted_prob = accepted_probs[dom_source]
dom_num = all_probs.index(accepted_prob)
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Dom source (%d)" % (dom_num + 1),
"Accept dom.\nsource",
len(matches),
plot_accept,
src_all,
"spectral",
i,
)
##If nothing dominates, send to check if a combined source works
else:
comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(
src_all, src_g, accepted_inds, "plot=no", len(matches)
)
##See whether combined or split
if "split" in comb_crit:
accept_type = "split"
else:
accept_type = "combine"
##Plot the combine or split, based on whether accepted or retained to investigate
if "Accepted" in comb_crit:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"No dom. source",
comb_crit,
len(matches),
plot_accept,
src_all,
accept_type,
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"No dom. source",
comb_crit,
len(matches),
plot_eyeball,
src_all,
accept_type,
i,
)
return i
def single_match_test(src_all, comp, accepted_matches, accepted_inds, g_stats, num_matches, repeated_cats, matches, i):
"""Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up"""
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([match])
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob) + 1
match_crit = "Combination (%d)\npossible\n%s repeated cats" % (dom_num, repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr = np.pi / 180.0
delta_RA = (
np.arccos((np.cos(closeness * dr) - np.sin(src_all.decs[0] * dr) ** 2) / np.cos(src_all.decs[0] * dr) ** 2) / dr
)
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra != -100000.0]
dec_test = [dec for dec in src_g.decs if dec != -100000.0]
small_test = []
for ra, dec in zip(ras_test, dec_test):
##We set up delta_RA to give us the equivalent offset in RA that corresponds to the
##resolution, so we use the offset in RA, not the arcdistance
prim_ra = src_all.ras[0]
ra_dist = ra - prim_ra
##Code to cope if one source 359.9, other 0.1 etc.
if abs(ra_dist) > 180.0:
if ra > 180.0:
ra -= 360.0
ra_dist = prim_ra - ra
else:
prim_ra -= 360.0
ra_dist = ra - prim_ra
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist / ra_axis) ** 2 + (dec_dist / dec_axis) ** 2
if ell_test <= 1:
small_test.append("yes")
else:
##Otherwise, fails
small_test.append("no")
# no_names.append(repeat_name) #Note the name of the sources that are far away
# Fail the positional test if a source is outside of the resolution plus position error
close_test = "passed"
if "no" in small_test:
close_test = "failed"
##If prob is higher than threshold, ignore position of sources and accept the match
if prob > high_prob:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"N/A",
"Pos. accepted\nby $P>P_u$",
num_matches,
plot_accept,
src_all,
"position",
i,
)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test == "passed" or prob > low_prob:
##IF below either threshold, append with the applicable fit label
if jstat_resids[0] <= jstat_thresh or chi_resids[0] <= chi_thresh:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Spec. passed",
"Accept by spec",
num_matches,
plot_accept,
src_all,
"spectral",
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Spec. failed",
"Reject by spec",
num_matches,
plot_reject,
src_all,
"spectral",
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"N/A",
"pos reject by $P<P_l$",
num_matches,
plot_reject,
src_all,
"position",
i,
)
return i
def create_plot(comp,accepted_inds,match_crit,dom_crit,outcome):
'''The main plotting function that takes the relevant data and plots the outcome'''
###Split the information up as needed
chunks = comp.split('START_COMP')
all_info = chunks[0].split('\n')
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry=='': del all_info[all_info.index(entry)]
for entry in all_info:
if 'START' in entry: del all_info[all_info.index(entry)]
matches = chunks[1].split('\n')
del matches[0],matches[-2:]
##See how many matches there are, and set up the number of plots needed. If there are
##more than 16 matches, only plot the top 15 and print how many more matches there
##were - saves heaps of tiny little plots from appearing
num_matches = len(matches)
skip_16 = 'no'
if num_matches==1:
width = 1
height = 2
elif num_matches>16:
width = 4
height = 4
skip_16 = 'yes'
else:
width = int(num_matches**0.5)
height = num_matches/width
if num_matches%width==0:
pass
else:
height+=1
##Sets up a grid layout for the whole of the figure. We'll use half later on for
##the individual plots
gs = gridspec.GridSpec(int(round(height)),2*width)
##Need a big plot!
fig = plt.figure(figsize = (18,11))
##Find out the ra,dec of the base catalogue source
info=all_info[0].split()
ra_main = float(info[2])
dec_main = float(info[4])
##Set up dedicated left plots
main_dims = [0.15, 0.5, 0.29, 0.35]
spec_dims = [0.15, 0.1, 0.29, 0.35]
ax_main,ax_spectral,tr_fk5,wcs = make_left_plots(fig,main_dims,spec_dims,ra_main,dec_main)
##Find the limits out to search area - have to do each edge individual,
##because of the arcdistance projection malarky
##Even at the same dec, 3 arcmins apart in RA doesn't translate to 3arcmin arcdist - projection
##fun. Can use law of cosines on a sphere to work out appropriate delta RA. Use this to define plot
##limits for a nice looking plot
delta_RA = np.arccos((np.cos((2*closeness)*dr)-np.sin(dec_main*dr)**2)/np.cos(dec_main*dr)**2)/dr
plot_lim = (2*closeness) + (0.1/60.0)
ra_up_lim = ra_main + delta_RA + (0.1/60.0)
ra_down_lim = ra_main - delta_RA - (0.1/60.0)
dec_up_lim = dec_main + plot_lim
dec_down_lim = dec_main - plot_lim
###Plot the individual combination plots - do this first so the error bars go over
##the top of the line plots
spec_labels = []
SIs = []
for i in np.arange(height):
for j in range(width,2*width):
if i*j == 21:
if skip_16=='yes':
ax = plt.subplot(gs[int(round(i)),int(round(j))])
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.text(0.5,0.5,"And %d\nother\nplots" %(num_matches-15),transform=ax.transAxes,verticalalignment='center',horizontalalignment='center',fontsize=16)
else:
try:
ind = (i*width)+(j-width)
match = matches[int(round(ind))]
ax = plt.subplot(gs[int(round(i)),int(round(j))])
ax.set_xticklabels([])
ax.set_yticklabels([])
##TODO - if plot centred on or close to RA,Dec = 0,0 then going to get wrapping problems. Should be able to pull the need
##for a wrap from ra_down_lim,ra_up_lim - one should be <0.0, or >360.0. Need to happen inside plot_ind
prob,resid,spec_plot,params,bse = plot_ind(match,ax,ind,ax_spectral,ra_down_lim,ra_up_lim,dec_down_lim,dec_up_lim,dom_crit,outcome)
if spec_plot=='na':
pass
else:
SI_err = '%.2f' %bse[0]
if SI_err == 'inf': SI_err = 'N/A'
SIs.append([params[0],SI_err,str(ind+1)])
spec_labels.append(spec_plot)
except IndexError:
pass
#===========================================================#
##Plot the matching criteria information
match1 = matches[0].split()
src_g = mkl.get_srcg(match1)
#spec_leg = fig.add_axes([0.435,0.1,0.125,0.35])
text_axes = fig.add_axes([0.445,0.5,0.125,0.35])
text_axes.axis('off')
##Plot the matching information
props = dict(boxstyle='round', facecolor='w',lw='1.5')
text_axes.text(0.5,0.5,'Match Criteria:\n%s\n\nDominace Test:\n%s\n\nOutcome:\n%s' %(match_crit,dom_crit,outcome),
bbox=props,transform=text_axes.transAxes,verticalalignment='center',horizontalalignment='center',fontsize=16)
all_fluxs = []
##Fill the left hand plots with information goodness
all_freqs = fill_left_plots(all_info,ra_main,dec_main,ax_main,ax_spectral,tr_fk5,wcs,all_fluxs,ra_down_lim,ra_up_lim,dec_down_lim,dec_up_lim,delta_RA)
##If no repeated catalogues to combine, skip
if num_matches==0 or num_matches==1:
pass
##Otherwise, plot the combined fluxes
else:
##Calculate and plot the combined fluxes of the two sources, even if source one or two has been accepted
##just as a guide
src_all = mkl.get_allinfo(all_info)
##If positionally impossible don't plot combined info
if accepted_inds=='Nope':
pass
##If only one position possible, don't plot combined info
elif len(accepted_inds) == 1:
pass
##Otherwise, see what the combined fluxes look like
else:
comb_crit, ra_ws, rerr_ws, dec_ws, derr_ws, temp_freqs, comb_freqs, comb_fluxs, comb_ferrs, comb_fit, comb_jstat, comb_chi_red, comb_bse, combined_names, set_freqs, set_fluxs, set_fits, set_bses = mkl.combine_flux(src_all,src_g,accepted_inds,'plot=yes',len(matches))
##If the criteria sent the double to be combined, actually plot the fitted line
if dom_crit == 'No dom. source':
for freq,flux in zip(set_freqs,set_fluxs):
ax_spectral.plot(freq,flux,linestyle='--',linewidth=1,color='r')
split_colors = ['#AE70ED','#FFB60B','#62A9FF','#59DF00']
for fit,bse in zip(set_fits,set_bses):
ind = set_fits.index(fit)
ax_spectral.plot(set_freqs[ind],set_fluxs[ind],linestyle='--',linewidth=1.0,color=split_colors[ind],alpha=0.7)
split_p, = ax_spectral.plot(temp_freqs,np.exp(fit.params[1] + np.log(temp_freqs)*fit.params[0]),linestyle='-',linewidth=1.5,color=split_colors[ind])
spec_labels.append(split_p)
SI_err = '%.2f' %bse[0]
if SI_err == 'inf': SI_err = 'N/A'
SIs.append([fit.params[0],SI_err,'split %d' %(ind+1)])
bright_colours = ['#FF6600','#33FF33','#FF47A3','#00ebb3']
for freq in np.arange(len(comb_freqs)):
plot_errors_comb('*',bright_colours[freq],comb_freqs[freq],comb_fluxs[freq],comb_ferrs[freq],'combo',16,ax_spectral)
comb_p, = ax_spectral.plot(temp_freqs,np.exp(comb_fit.fittedvalues),linestyle='--',linewidth=1.5,color='k')
spec_labels.append(comb_p)
SI_err = '%.2f' %comb_bse[0]
if SI_err == 'inf': SI_err = 'N/A'
SIs.append([comb_fit.params[0],SI_err,'comb'])
##Send the combined fluxes to the all_fluxs so that ax_spectral is scaled appropriately
for flux in comb_fluxs:
all_fluxs.append(flux)
for pos in np.arange(len(ra_ws)):
patch = plot_pos_comb('*',bright_colours[pos],ra_ws[pos],dec_ws[pos],rerr_ws[pos],derr_ws[pos],combined_names[pos],16,ax_main,ax_main.get_transform("fk5"))
scale_spectral(all_fluxs,all_freqs,ax_spectral)
##==============================================================
fig.tight_layout()
fig.subplots_adjust(bottom=0.1)
fig.subplots_adjust(left=0.15)
##Make room at the top of the plot for a legend for ax_main, make the legend
fig.subplots_adjust(top=0.85)
leg_labels = [r'$\alpha_{%s}$ = %.2f$\pm$%s' %(SI[2],SI[0],SI[1]) for SI in SIs]
main_handles,main_labels = ax_main.get_legend_handles_labels()
main_leg = fig.add_axes([0.05,0.87,0.9,0.05])
main_leg.axis('off')
main_leg.legend(main_handles,main_labels,loc='lower center',prop={'size':12},ncol=8) #,bbox_to_anchor=(0,1.02),
spec_leg = fig.add_axes([0.445,0.1,0.125,0.35])
spec_leg.axis('off')
##Stop the legend from having so many entries that it goes off the plot
if len(spec_labels)>11:
trim_labels = spec_labels[:10]
trim_labels.append(spec_labels[-1])
trim_legs = leg_labels[:10]
trim_legs.append(leg_labels[-1])
spec_leg.legend(trim_labels,trim_legs,loc='center',prop={'size':14},fancybox=True)
else:
spec_leg.legend(spec_labels,leg_labels,loc='center',prop={'size':14},fancybox=True)
##Create an axes to contain patches for an ellipse legend
patch_leg = fig.add_axes([0.015,0.1,0.06,0.75])
patch_leg.set_xticks([])
patch_leg.set_yticks([])
patch_leg.set_xticklabels([])
patch_leg.set_yticklabels([])
##See what catalogues are present in the match
present_cats = [cat for cat in set(src_g.cats) if cat!='-100000.0']
##Scale accordingly
increment = 1.0/(2+len(present_cats))
ell_positions = np.arange(increment/2,1,increment)
##Find the axes coord transform
patch_trans = patch_leg.transAxes
##Plot and name the resolution ellipse
ell = patches.Ellipse((0.5,ell_positions[-2]),0.9,increment-0.05,angle=0,
transform=patch_trans, linestyle='dashed',fc='none',lw=1.1,color='gray')
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[-2],'Resolution\n+ error',
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=14)
##Plot and name the search ellipse
ell = patches.Ellipse((0.5,ell_positions[-1]),0.9,increment-0.05,angle=0,
transform=patch_trans, linestyle='dashdot',fc='none',lw=1.1,color='k')
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[-1],'Search\nradius',
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=14)
##Use the same method as plot_all - for some reason was getting transform errors.
##so do it separately here (sigh)
for cat in present_cats:
col_ind = matched_cats.index(cat)
position_ind = present_cats.index(cat)
patch_leg.errorbar(0.5,ell_positions[position_ind],0.01,0.075,marker=markers[col_ind],ms=8,mfc=marker_colours[col_ind],
mec=marker_colours[col_ind],ecolor=marker_colours[col_ind],markeredgewidth=1,label='meh',linestyle='None',transform=patch_trans)
ell = patches.Ellipse((0.5,ell_positions[position_ind]),0.9,increment-0.05,angle=0, transform=patch_trans,
fc=ell_colours1[col_ind],color=ell_colours2[col_ind],alpha=alphas[col_ind])
patch_leg.add_patch(ell)
patch_leg.text(0.5,ell_positions[position_ind]-(increment/2-0.04),cat,
transform=patch_trans,verticalalignment='center',horizontalalignment='center',fontsize=16)
if save_plots:
plt.savefig('%s-pumaplot.png' %all_info[0].split()[1],bbox_inches='tight',dpi=75)
plt.close()
else:
plt.show()
##Get the information into nice usable forms, and get rid of empty/pointless
##entries
chunks = comp.split('START_COMP')
all_info = chunks[0].split('\n')
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry=='': del all_info[all_info.index(entry)]
for entry in all_info:
if 'START' in entry: del all_info[all_info.index(entry)]
matches = chunks[1].split('\n')
del matches[0],matches[-1]
stats = matches[-1]
del matches[-2:]
match1 = matches[0].split()
src_g = mkl.get_srcg(match1)
##Get some info and find which catalogues are present
src_all = mkl.get_allinfo(all_info)
try:
if src_all.names[0] in extended_names:
present_cats = [cat for cat in src_all.cats if cat!='-100000.0']
meh,num_matches,accept_matches,accepted_inds,accept_type,stage = stats.split()
image_num = bayes_comp.index(comp)+1
image_locs = "./extended_fits"
image_cats = [cat for cat in ['gleam','vlssr','sumss','nvss'] if os.path.exists("%s/%s_%s.fits" %(image_locs,src_all.names[0],cat))==True]
image_files =["%s/%s_%s.fits" %(image_locs,src_all.names[0],cat) for cat in ['gleam','vlssr','sumss','nvss'] if os.path.exists("%s/%s_%s.fits" %(image_locs,src_all.names[0],cat))==True]
fig = do_plot_image(all_info,image_cats,image_files,present_cats,src_all.names,src_g,matches)
def make_plots(comp, i):
##Get the information into nice usable forms, and get rid of empty/pointless
##entries
chunks = comp.split('START_COMP')
all_info = chunks[0].split('\n')
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry == '': del all_info[all_info.index(entry)]
for entry in all_info:
if 'START' in entry: del all_info[all_info.index(entry)]
matches = chunks[1].split('\n')
del matches[0], matches[-2:]
##Put the information for every source in the matched group in one source_group() class
##(see apply_criteria_lib for source_group())
src_all = mkl.get_allinfo(all_info)
#print src_all.names[0]
##This line applies positional criteria, and tells us if a simple one catalogue repeat source, returning
##how many combinations are possible and statistics on them
repeated_cats, accepted_matches, accepted_inds, accepted_probs, jstats, chi_reds, g_stats = mkl.matches_retained(
src_all, matches)
match_crit = "%d of %d \ncombinations \npossible \n%s repeated cats" % (
len(accepted_matches), len(matches), repeated_cats)
##If no combinations are possible, reject all info (goes into the eyeball document)
if len(accepted_matches) == 0:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'Positionally\nimpossible', 'N/A',
len(matches), plot_reject, src_all, 'position', i)
##If just one combo positionally possible, do a single combo check
elif len(accepted_matches) == 1:
i = single_match_test(src_all, comp, accepted_matches, accepted_inds,
g_stats, len(matches), repeated_cats, matches, i)
##(Any plotting gets done within single_match_test)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != '-100000.0']))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs,
num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != 'none':
jstat_resids, params, bses, chi_resids = mkl.calculate_resids(
[accepted_matches[dom_source]])
##Find the probs of all the matches, and use the prob of the dom match to see what number match was accepted
all_probs = [float(match.split()[-1]) for match in matches]
accepted_prob = accepted_probs[dom_source]
dom_num = all_probs.index(accepted_prob)
i = plot_accept_type(comp, accepted_inds, match_crit,
'Dom source (%d)' % (dom_num + 1),
'Accept dom.\nsource', len(matches),
plot_accept, src_all, 'spectral', i)
##If nothing dominates, send to check if a combined source works
else:
comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(
src_all, src_g, accepted_inds, 'plot=no', len(matches))
##See whether combined or split
if 'split' in comb_crit:
accept_type = 'split'
else:
accept_type = 'combine'
##Plot the combine or split, based on whether accepted or retained to investigate
if 'Accepted' in comb_crit:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'No dom. source', comb_crit,
len(matches), plot_accept, src_all,
accept_type, i)
else:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'No dom. source', comb_crit,
len(matches), plot_eyeball, src_all,
accept_type, i)
return i
def single_match_test(src_all,comp,accepted_matches,accepted_inds,g_stats,num_matches,repeated_cats,matches):
'''Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up'''
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids,params,bses,chi_resids = mkl.calculate_resids([match])
##Gather source information
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob)+1
match_crit = "Combination (%d)\npossible\n%s repeated cats" %(dom_num,repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr=np.pi/180.0
delta_RA = np.arccos((np.cos(closeness*dr)-np.sin(src_all.decs[0]*dr)**2)/np.cos(src_all.decs[0]*dr)**2)/dr
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra!=-100000.0]
dec_test = [dec for dec in src_g.decs if dec!=-100000.0]
small_test = []
for ra,dec in zip(ras_test,dec_test):
##Even though at same dec, 3arcmis offset in RA isn't neccessarily 3arcmins arcdistance
ra_dist = mkl.arcdist(src_all.ras[0],ra,src_all.decs[0],src_all.decs[0])
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist/ra_axis)**2 + (dec_dist/dec_axis)**2
if ell_test <= 1:
small_test.append('yes')
else:
##Otherwise, fails
small_test.append('no')
#no_names.append(repeat_name) #Note the name of the sources that are far away
#Fail the positional test if a source is outside of the resolution plus position error
close_test = 'passed'
if 'no' in small_test: close_test = 'failed'
##If prob is higher than threshold, ignore position of sources and accept the match
if prob>high_prob:
##Accept the source, put it in a way that can be read when constructing the final table
if chi_resids[0]<=2:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'position',0,chi_resids[0])
else:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'position',1,chi_resids[0])
make_accept(comp,g_stats,'position',accepted_inds)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test=='passed' or prob>low_prob:
##IF below eith threshold, append with the applicable fit label
if jstat_resids[0]<=jstat_thresh or chi_resids[0]<=chi_thresh:
if chi_resids[0]<=2:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',0,chi_resids[0])
else:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',1,chi_resids[0])
make_accept(comp,g_stats,'spectral',accepted_inds)
else:
g_stats.retained_matches = 1
##Put accepted inds as [0] just to have something outputted to the investigate text file -
##accepted_inds is empty is rejecting at this stage
make_rejection(comp,g_stats,'spectral',[0])
else:
g_stats.retained_matches = 1
make_rejection(comp,g_stats,'position',[0])
repeated_inds = [i for i in xrange(len(cats)) if cats.count(cats[i])>1] make_rejection(comp,g_stats,'position',repeated_inds) ##If just one combo positionally possible, do a single combo check elif len(accepted_matches)==1: single_match_test(src_all,comp,accepted_matches,accepted_inds,g_stats,len(matches),repeated_cats,matches) ##If more than one combination is positionally possible: else: ##Check for a dominant source. The combination must be the only one with high position prob, ##all others with low positional proability, and must dominate spectrally num_cat = len(set([cat for cat in src_all.cats if cat!='-100000.0'])) dom_source = mkl.spec_pos_agree(jstats,chi_reds,accepted_probs,num_cat) src_g = mkl.get_srcg(accepted_matches[0]) ##If it finds a dominant source, accept it - counts as a spectral match if dom_source!='none': jstat_resids,params,bses,chi_resids = mkl.calculate_resids([accepted_matches[dom_source]]) if chi_resids[0]<=2: make_entry(accepted_matches[dom_source],params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',0,chi_resids[0]) else: make_entry(accepted_matches[dom_source],params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',1,chi_resids[0]) make_accept(comp,g_stats,'spectral',accepted_inds) ##If nothing dominates, send to check if a combined source works else: comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(src_all,src_g,accepted_inds,'plot=no',len(matches)) if 'Accepted' in comb_crit: ##If source was combined, add one new source with combine in the g_stat if len(comb_source)==1:
