def single_match_test(src_all, comp, accepted_matches, accepted_inds, g_stats,
num_matches, repeated_cats, matches):
'''Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up'''
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([match])
##Gather source information
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob) + 1
match_crit = "Combination (%d)\npossible\n%s repeated cats" % (
dom_num, repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr = np.pi / 180.0
delta_RA = np.arccos(
(np.cos(closeness * dr) - np.sin(src_all.decs[0] * dr)**2) /
np.cos(src_all.decs[0] * dr)**2) / dr
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra != -100000.0]
dec_test = [dec for dec in src_g.decs if dec != -100000.0]
small_test = []
for ra, dec in zip(ras_test, dec_test):
##We set up delta_RA to give us the equivalent offset in RA that corresponds to the
##resolution, so we use the offset in RA, not the arcdistance
prim_ra = src_all.ras[0]
ra_dist = ra - prim_ra
##Code to cope if one source 359.9, other 0.1 etc.
if abs(ra_dist) > 180.0:
if ra > 180.0:
ra -= 360.0
ra_dist = prim_ra - ra
else:
prim_ra -= 360.0
ra_dist = ra - prim_ra
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist / ra_axis)**2 + (dec_dist / dec_axis)**2
if ell_test <= 1:
small_test.append('yes')
else:
##Otherwise, fails
small_test.append('no')
#no_names.append(repeat_name) #Note the name of the sources that are far away
#Fail the positional test if a source is outside of the resolution plus position error
close_test = 'passed'
if 'no' in small_test: close_test = 'failed'
##If prob is higher than threshold, ignore position of sources and accept the match
if prob > high_prob:
##Accept the source, put it in a way that can be read when constructing the final table
if chi_resids[0] <= 2:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'position', 0, chi_resids[0],
'isolated')
else:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'position', 1, chi_resids[0],
'isolated')
make_accept(comp, g_stats, 'position', accepted_inds)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test == 'passed' or prob > low_prob:
##IF below eith threshold, append with the applicable fit label
if jstat_resids[0] <= jstat_thresh or chi_resids[0] <= chi_thresh:
if chi_resids[0] <= 2:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'spectral', 0,
chi_resids[0], 'isolated')
else:
make_entry(match, params[0][0], params[0][1], bses[0][0],
bses[0][1], g_stats, 'spectral', 1,
chi_resids[0], 'isolated')
make_accept(comp, g_stats, 'spectral', accepted_inds)
else:
g_stats.retained_matches = 1
##Put accepted inds as [0] just to have something outputted to the investigate text file -
##accepted_inds is empty is rejecting at this stage
make_rejection(comp, g_stats, 'spectral', [0])
else:
g_stats.retained_matches = 1
make_rejection(comp, g_stats, 'position', [0])
# reject_pile.write('END_GROUP\n')
elif answer == 'a':
if answer_2 == 'c':
comb_sources = combine_flux(src_all,src_g,map(int,accepted_inds.split(',')),num_matches)
comb_sources[0].inspected = 1
comb_sources[0].accept = accept_type
sources.append(comb_sources[0])
#g_stats.accept_type = 'check-combine'
sources_stats.append(g_stats)
#break
else:
accepted_match = matches[int(answer_2)-1].split()
jstat_resids,params,bses,chi_resids = mkl.calculate_resids([accepted_match])
source = get_srcg(accepted_match)
source.SI = params[0][0]
source.intercept = params[0][1]
source.SI_err = bses[0][0]
source.intercept_err = bses[0][1]
source.chi_resid = chi_resids[0]
source.epsilon_red = jstat_resids[0]
source.inspected = 1
if chi_resids[0]<=2:
source.low_resids = 0
else:
source.low_resids = 1
for cat,name in zip(source.cats[1:],source.names[1:]):
if cat == 'vlssr':
single_match_test(src_all, comp, accepted_matches, accepted_inds,
g_stats, len(matches), repeated_cats, matches)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != '-100000.0']))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs,
num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != 'none':
jstat_resids, params, bses, chi_resids = mkl.calculate_resids(
[accepted_matches[dom_source]])
if chi_resids[0] <= 2:
make_entry(accepted_matches[dom_source], params[0][0],
params[0][1], bses[0][0], bses[0][1], g_stats,
'spectral', 0, chi_resids[0], 'dominant')
else:
make_entry(accepted_matches[dom_source], params[0][0],
params[0][1], bses[0][0], bses[0][1], g_stats,
'spectral', 1, chi_resids[0], 'dominant')
make_accept(comp, g_stats, 'spectral', accepted_inds)
##If nothing dominates, send to check if a combined source works
else:
comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(
src_all, src_g, accepted_inds, 'plot=no', len(matches))
if 'Accepted' in comb_crit:
def make_plots(comp, i):
##Get the information into nice usable forms, and get rid of empty/pointless
##entries
chunks = comp.split("START_COMP")
all_info = chunks[0].split("\n")
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry == "":
del all_info[all_info.index(entry)]
for entry in all_info:
if "START" in entry:
del all_info[all_info.index(entry)]
matches = chunks[1].split("\n")
del matches[0], matches[-2:]
##Put the information for every source in the matched group in one source_group() class
##(see apply_criteria_lib for source_group())
src_all = mkl.get_allinfo(all_info)
# print src_all.names[0]
##This line applies positional criteria, and tells us if a simple one catalogue repeat source, returning
##how many combinations are possible and statistics on them
repeated_cats, accepted_matches, accepted_inds, accepted_probs, jstats, chi_reds, g_stats = mkl.matches_retained(
src_all, matches
)
match_crit = "%d of %d \ncombinations \npossible \n%s repeated cats" % (
len(accepted_matches),
len(matches),
repeated_cats,
)
##If no combinations are possible, reject all info (goes into the eyeball document)
if len(accepted_matches) == 0:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Positionally\nimpossible",
"N/A",
len(matches),
plot_reject,
src_all,
"position",
i,
)
##If just one combo positionally possible, do a single combo check
elif len(accepted_matches) == 1:
i = single_match_test(
src_all, comp, accepted_matches, accepted_inds, g_stats, len(matches), repeated_cats, matches, i
)
##(Any plotting gets done within single_match_test)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != "-100000.0"]))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs, num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != "none":
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([accepted_matches[dom_source]])
##Find the probs of all the matches, and use the prob of the dom match to see what number match was accepted
all_probs = [float(match.split()[-1]) for match in matches]
accepted_prob = accepted_probs[dom_source]
dom_num = all_probs.index(accepted_prob)
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Dom source (%d)" % (dom_num + 1),
"Accept dom.\nsource",
len(matches),
plot_accept,
src_all,
"spectral",
i,
)
##If nothing dominates, send to check if a combined source works
else:
comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(
src_all, src_g, accepted_inds, "plot=no", len(matches)
)
##See whether combined or split
if "split" in comb_crit:
accept_type = "split"
else:
accept_type = "combine"
##Plot the combine or split, based on whether accepted or retained to investigate
if "Accepted" in comb_crit:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"No dom. source",
comb_crit,
len(matches),
plot_accept,
src_all,
accept_type,
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"No dom. source",
comb_crit,
len(matches),
plot_eyeball,
src_all,
accept_type,
i,
)
return i
def single_match_test(src_all, comp, accepted_matches, accepted_inds, g_stats, num_matches, repeated_cats, matches, i):
"""Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up"""
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids, params, bses, chi_resids = mkl.calculate_resids([match])
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob) + 1
match_crit = "Combination (%d)\npossible\n%s repeated cats" % (dom_num, repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr = np.pi / 180.0
delta_RA = (
np.arccos((np.cos(closeness * dr) - np.sin(src_all.decs[0] * dr) ** 2) / np.cos(src_all.decs[0] * dr) ** 2) / dr
)
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra != -100000.0]
dec_test = [dec for dec in src_g.decs if dec != -100000.0]
small_test = []
for ra, dec in zip(ras_test, dec_test):
##We set up delta_RA to give us the equivalent offset in RA that corresponds to the
##resolution, so we use the offset in RA, not the arcdistance
prim_ra = src_all.ras[0]
ra_dist = ra - prim_ra
##Code to cope if one source 359.9, other 0.1 etc.
if abs(ra_dist) > 180.0:
if ra > 180.0:
ra -= 360.0
ra_dist = prim_ra - ra
else:
prim_ra -= 360.0
ra_dist = ra - prim_ra
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist / ra_axis) ** 2 + (dec_dist / dec_axis) ** 2
if ell_test <= 1:
small_test.append("yes")
else:
##Otherwise, fails
small_test.append("no")
# no_names.append(repeat_name) #Note the name of the sources that are far away
# Fail the positional test if a source is outside of the resolution plus position error
close_test = "passed"
if "no" in small_test:
close_test = "failed"
##If prob is higher than threshold, ignore position of sources and accept the match
if prob > high_prob:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"N/A",
"Pos. accepted\nby $P>P_u$",
num_matches,
plot_accept,
src_all,
"position",
i,
)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test == "passed" or prob > low_prob:
##IF below either threshold, append with the applicable fit label
if jstat_resids[0] <= jstat_thresh or chi_resids[0] <= chi_thresh:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Spec. passed",
"Accept by spec",
num_matches,
plot_accept,
src_all,
"spectral",
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"Spec. failed",
"Reject by spec",
num_matches,
plot_reject,
src_all,
"spectral",
i,
)
else:
i = plot_accept_type(
comp,
accepted_inds,
match_crit,
"N/A",
"pos reject by $P<P_l$",
num_matches,
plot_reject,
src_all,
"position",
i,
)
return i
def make_plots(comp, i):
##Get the information into nice usable forms, and get rid of empty/pointless
##entries
chunks = comp.split('START_COMP')
all_info = chunks[0].split('\n')
##FOR SOME REASON CAN'T DO BOTH OF THESE LINES IN THE SAME FOR LOOP?!?!?!
for entry in all_info:
if entry == '': del all_info[all_info.index(entry)]
for entry in all_info:
if 'START' in entry: del all_info[all_info.index(entry)]
matches = chunks[1].split('\n')
del matches[0], matches[-2:]
##Put the information for every source in the matched group in one source_group() class
##(see apply_criteria_lib for source_group())
src_all = mkl.get_allinfo(all_info)
#print src_all.names[0]
##This line applies positional criteria, and tells us if a simple one catalogue repeat source, returning
##how many combinations are possible and statistics on them
repeated_cats, accepted_matches, accepted_inds, accepted_probs, jstats, chi_reds, g_stats = mkl.matches_retained(
src_all, matches)
match_crit = "%d of %d \ncombinations \npossible \n%s repeated cats" % (
len(accepted_matches), len(matches), repeated_cats)
##If no combinations are possible, reject all info (goes into the eyeball document)
if len(accepted_matches) == 0:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'Positionally\nimpossible', 'N/A',
len(matches), plot_reject, src_all, 'position', i)
##If just one combo positionally possible, do a single combo check
elif len(accepted_matches) == 1:
i = single_match_test(src_all, comp, accepted_matches, accepted_inds,
g_stats, len(matches), repeated_cats, matches, i)
##(Any plotting gets done within single_match_test)
##If more than one combination is positionally possible:
else:
##Check for a dominant source. The combination must be the only one with high position prob,
##all others with low positional proability, and must dominate spectrally
num_cat = len(set([cat for cat in src_all.cats if cat != '-100000.0']))
dom_source = mkl.spec_pos_agree(jstats, chi_reds, accepted_probs,
num_cat)
src_g = mkl.get_srcg(accepted_matches[0])
##If it finds a dominant source, accept it - counts as a spectral match
if dom_source != 'none':
jstat_resids, params, bses, chi_resids = mkl.calculate_resids(
[accepted_matches[dom_source]])
##Find the probs of all the matches, and use the prob of the dom match to see what number match was accepted
all_probs = [float(match.split()[-1]) for match in matches]
accepted_prob = accepted_probs[dom_source]
dom_num = all_probs.index(accepted_prob)
i = plot_accept_type(comp, accepted_inds, match_crit,
'Dom source (%d)' % (dom_num + 1),
'Accept dom.\nsource', len(matches),
plot_accept, src_all, 'spectral', i)
##If nothing dominates, send to check if a combined source works
else:
comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(
src_all, src_g, accepted_inds, 'plot=no', len(matches))
##See whether combined or split
if 'split' in comb_crit:
accept_type = 'split'
else:
accept_type = 'combine'
##Plot the combine or split, based on whether accepted or retained to investigate
if 'Accepted' in comb_crit:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'No dom. source', comb_crit,
len(matches), plot_accept, src_all,
accept_type, i)
else:
i = plot_accept_type(comp, accepted_inds,
match_crit, 'No dom. source', comb_crit,
len(matches), plot_eyeball, src_all,
accept_type, i)
return i
def single_match_test(src_all,comp,accepted_matches,accepted_inds,g_stats,num_matches,repeated_cats,matches):
'''Takes a combination of sources, one from each catalogue, with positional probabilities,
and determines whether they are a match or not - Algorithm 2 in the write up'''
match = accepted_matches[0]
prob = float(match[-1])
##calculate_resids needs a list of matches - calculate parameters
jstat_resids,params,bses,chi_resids = mkl.calculate_resids([match])
##Gather source information
src_g = mkl.get_srcg(match)
##Play the prob trick again to work out which match has been accepted
match_probs = [float(m.split()[-1]) for m in matches]
dom_num = match_probs.index(prob)+1
match_crit = "Combination (%d)\npossible\n%s repeated cats" %(dom_num,repeated_cats)
##Check to see if all matched sources are within the closeness test - create an
##error ellipse by combined closeness with base cat error
##Need to convert closeness in to an RA offset, due to spherical trigonometry
dr=np.pi/180.0
delta_RA = np.arccos((np.cos(closeness*dr)-np.sin(src_all.decs[0]*dr)**2)/np.cos(src_all.decs[0]*dr)**2)/dr
##Make a list of the ras and decs of the sources to distance test
ras_test = [ra for ra in src_g.ras if ra!=-100000.0]
dec_test = [dec for dec in src_g.decs if dec!=-100000.0]
small_test = []
for ra,dec in zip(ras_test,dec_test):
##Even though at same dec, 3arcmis offset in RA isn't neccessarily 3arcmins arcdistance
ra_dist = mkl.arcdist(src_all.ras[0],ra,src_all.decs[0],src_all.decs[0])
dec_dist = src_all.decs[0] - dec
ra_axis = src_all.rerrs[0] + abs(delta_RA)
dec_axis = src_all.derrs[0] + closeness
##Test to see if the source lies with an error ellipse created using semi-major
##and minor axes defined by the ra and dec error of the base cat + half the resolution
##of the base cat (closeness)
ell_test = (ra_dist/ra_axis)**2 + (dec_dist/dec_axis)**2
if ell_test <= 1:
small_test.append('yes')
else:
##Otherwise, fails
small_test.append('no')
#no_names.append(repeat_name) #Note the name of the sources that are far away
#Fail the positional test if a source is outside of the resolution plus position error
close_test = 'passed'
if 'no' in small_test: close_test = 'failed'
##If prob is higher than threshold, ignore position of sources and accept the match
if prob>high_prob:
##Accept the source, put it in a way that can be read when constructing the final table
if chi_resids[0]<=2:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'position',0,chi_resids[0])
else:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'position',1,chi_resids[0])
make_accept(comp,g_stats,'position',accepted_inds)
else:
##look to see if all sources are within the resolution of the
##base catalogue or above some probability theshold, if so check with a spec test else reject them
if close_test=='passed' or prob>low_prob:
##IF below eith threshold, append with the applicable fit label
if jstat_resids[0]<=jstat_thresh or chi_resids[0]<=chi_thresh:
if chi_resids[0]<=2:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',0,chi_resids[0])
else:
make_entry(match,params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',1,chi_resids[0])
make_accept(comp,g_stats,'spectral',accepted_inds)
else:
g_stats.retained_matches = 1
##Put accepted inds as [0] just to have something outputted to the investigate text file -
##accepted_inds is empty is rejecting at this stage
make_rejection(comp,g_stats,'spectral',[0])
else:
g_stats.retained_matches = 1
make_rejection(comp,g_stats,'position',[0])
##If just one combo positionally possible, do a single combo check elif len(accepted_matches)==1: single_match_test(src_all,comp,accepted_matches,accepted_inds,g_stats,len(matches),repeated_cats,matches) ##If more than one combination is positionally possible: else: ##Check for a dominant source. The combination must be the only one with high position prob, ##all others with low positional proability, and must dominate spectrally num_cat = len(set([cat for cat in src_all.cats if cat!='-100000.0'])) dom_source = mkl.spec_pos_agree(jstats,chi_reds,accepted_probs,num_cat) src_g = mkl.get_srcg(accepted_matches[0]) ##If it finds a dominant source, accept it - counts as a spectral match if dom_source!='none': jstat_resids,params,bses,chi_resids = mkl.calculate_resids([accepted_matches[dom_source]]) if chi_resids[0]<=2: make_entry(accepted_matches[dom_source],params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',0,chi_resids[0]) else: make_entry(accepted_matches[dom_source],params[0][0],params[0][1],bses[0][0],bses[0][1],g_stats,'spectral',1,chi_resids[0]) make_accept(comp,g_stats,'spectral',accepted_inds) ##If nothing dominates, send to check if a combined source works else: comb_crit, comb_source, comb_jstat, comb_chi_red = mkl.combine_flux(src_all,src_g,accepted_inds,'plot=no',len(matches)) if 'Accepted' in comb_crit: ##If source was combined, add one new source with combine in the g_stat if len(comb_source)==1: sources.append(comb_source[0]) g_stats.accept_type = 'combine' ##Gather stats