print('fitting ' + str(prior.nsrc) + ' sources \n')
print('using ' + str(prior.snpix) + ' pixels')
from xidplus.numpyro_fit import LOFAR150
fit = LOFAR150.LOFAR_150(prior, num_samples=1000, num_warmup=1000)
samples = fit.get_samples()['src_f']
posterior = xidplus.posterior_numpyro(fit, [prior])
if os.path.exists(
'data/xidplus_results/pybdsf/xidplus_run_{}'.format(n)) == True:
()
else:
os.mkdir('data/xidplus_results/pybdsf/xidplus_run_{}'.format(n))
xidplus.save(
[prior], posterior,
'data/xidplus_results/pybdsf/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.pkl'
.format(n, n))
LOFAR_cat = cat.create_LOFAR_cat(posterior, prior, 0)
LOFAR_cat = Table.read(LOFAR_cat)
with serialize_method_as(LOFAR_cat, None):
registry.write(
LOFAR_cat,
'data/xidplus_results/pybdsf/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'
.format(n, n),
format='fits',
overwrite=True)
from xidplus.stan_fit import MIPS
priors[0].prior_bkg(0.0,1)
priors[0].get_pointing_matrix()
#priors[0].upper_lim_map()
#priors[0].prior_flux_upper=(priors[0].prior_flux_upper-10.0+0.02)/np.max(priors[0].prf)
fit=MIPS.MIPS_24(priors[0],iter=1000)
posterior=xidplus.posterior_stan(fit,priors)
outfile=output_folder+'Tile_'+str(tiles[taskid-1])+'_'+str(order)
posterior=xidplus.posterior_stan(fit,priors)
xidplus.save(priors,posterior,outfile)
post_rep_map=postmaps.replicated_maps(priors,posterior,nrep=2000)
Bayes_P24=postmaps.Bayes_Pval_res(priors[0],post_rep_map[0])
cat=catalogue.create_MIPS_cat(posterior, priors[0], Bayes_P24)
kept_sources=moc_routines.sources_in_tile([tiles[taskid-1]],order,priors[0].sra,priors[0].sdec)
kept_sources=np.array(kept_sources)
cat[1].data=cat[1].data[kept_sources]
outfile=output_folder+'Tile_'+str(tiles[taskid-1])+'_'+str(order)
cat.writeto(outfile+'_MIPS24_cat.fits',overwrite=True)
Bayesian_Pval=postmaps.make_Bayesian_pval_maps(priors[0],post_rep_map[0])
wcs_temp=wcs.WCS(priors[0].imhdu)
ra,dec=wcs_temp.wcs_pix2world(priors[0].sx_pix,priors[0].sy_pix,0)
kept_pixels=np.array(moc_routines.sources_in_tile([tiles[taskid-1]],order,ra,dec))
from astropy.io import ascii, fits import pylab as plt %matplotlib inline from astropy import wcs import numpy as np import xidplus from xidplus import moc_routines import pickle from xidplus import sed SEDs, df=sed.berta_templates() priors,posterior=xidplus.load(filename='./XID+SED_prior.pkl') import xidplus.stan_fit.SED as SPM fit=SPM.MIPS_PACS_SPIRE(priors,SEDs,chains=4,iter=1000) posterior=sed.posterior_sed(fit,priors,SEDs) xidplus.save(priors, posterior, 'test_SPM')
posterior = xidplus.posterior_stan(fit, [prior250, prior350, prior500])
priors = [prior250, prior350, prior500]
import xidplus.catalogue as cat
SPIRE_cat = cat.create_SPIRE_cat(posterior, priors[0], priors[1], priors[2])
SPIRE_cat = Table.read(SPIRE_cat)
mask = [SPIRE_cat['HELP_ID'][i] in ids for i in range(len(SPIRE_cat))]
SPIRE_cat = SPIRE_cat[mask]
if os.path.exists('data/fir/v0_7/SPIRE/xidplus_run_{}'.format(taskid)) == True:
()
else:
os.mkdir('data/fir/v0_7/SPIRE/xidplus_run_{}'.format(taskid))
xidplus.save([prior250, prior350, prior500], posterior,
'data/fir/v0_7/SPIRE/xidplus_run_{}/lofar_xidplus_fir_{}'.format(
taskid, taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
with serialize_method_as(SPIRE_cat, None):
registry.write(
SPIRE_cat,
'data/fir/v0_7/SPIRE/xidplus_run_{}/lofar_xidplus_fir_{}.fits'.format(
taskid, taskid),
format='fits',
overwrite=True)
#Table.write(SPIRE_cat,'data/fir/v0_7/SPIRE/xidplus_run_{}/lofar_xidplus_fir_{}.fits'.format(taskid,taskid),overwrite=True)
time1 = time.time()
print('total time taken = {}'.format(time1 - time0))
posterior = xidplus.posterior_stan(fit, [prior250])
priors = [prior250]
import xidplus.catalogue as cat
MIPS_cat = cat.create_MIPS_cat(posterior, priors[0], 0)
MIPS_cat = Table.read(MIPS_cat)
mask = [MIPS_cat['help_id'][i] in ids for i in range(len(MIPS_cat))]
MIPS_cat = MIPS_cat[mask]
if os.path.exists(
'data/fir/MIPS_no_lofar/xidplus_run_{}'.format(taskid)) == True:
()
else:
os.mkdir('data/fir/MIPS_no_lofar/xidplus_run_{}'.format(taskid))
xidplus.save(
[prior250], posterior,
'data/fir/MIPS_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}'.format(
taskid, taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
with serialize_method_as(MIPS_cat, None):
registry.write(
MIPS_cat,
'data/fir/MIPS_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}.fits'.
format(taskid, taskid),
format='fits',
overwrite=True)
#Table.write(MIPS_cat,'data/fir/MIPS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),overwrite=True)
posterior = xidplus.posterior_stan(fit, [prior100, prior160])
priors = [prior100, prior160]
import xidplus.catalogue as cat
PACS_cat = cat.create_PACS_cat(posterior, priors[0], priors[1])
PACS_cat = Table.read(PACS_cat)
mask = [PACS_cat['help_id'][i] in ids for i in range(len(PACS_cat))]
PACS_cat = PACS_cat[mask]
if os.path.exists(
'data/fir/PACS_no_lofar/xidplus_run_{}'.format(taskid)) == True:
()
else:
os.mkdir('data/fir/PACS_no_lofar/xidplus_run_{}'.format(taskid))
xidplus.save(
[prior100, prior160], posterior,
'data/fir/PACS_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}'.format(
taskid, taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
with serialize_method_as(PACS_cat, None):
registry.write(
PACS_cat,
'data/fir/PACS_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}.fits'.
format(taskid, taskid),
format='fits',
overwrite=True)
#Table.write(PACS_cat,'data/fir/PACS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),overwrite=True)
print('fitting ' + str(prior.nsrc) + ' sources \n')
print('using ' + str(prior.snpix) + ' pixels')
from xidplus.numpyro_fit import LOFAR150
fit = LOFAR150.LOFAR_150(prior, num_samples=1000, num_warmup=1000)
samples = fit.get_samples()['src_f']
posterior = xidplus.posterior_numpyro(fit, [prior])
if os.path.exists(
'data/xidplus_results/pybdsf_components/xidplus_run_{}'.format(
n)) == True:
()
else:
os.mkdir('data/xidplus_results/pybdsf_components/xidplus_run_{}'.format(n))
xidplus.save(
[prior], posterior,
'data/xidplus_results/pybdsf_components/xidplus_run_{}/LOFAR_{}_run_components_restoring_beam.pkl'
.format(n, n))
LOFAR_cat = cat.create_LOFAR_cat(posterior, prior)
LOFAR_cat = Table.read(LOFAR_cat)
with serialize_method_as(LOFAR_cat, None):
registry.write(
LOFAR_cat,
'data/xidplus_results/pybdsf_components/xidplus_run_{}/LOFAR_{}_run_components_restoring_beam'
.format(n, n),
format='fits',
overwrite=True)
else:
prfs.append(psf_dirty[0])
pinds.append(np.arange(0,prfs[m].shape[0],1))
prior.set_prfs(prfs,pinds,pinds)
prior.get_pointing_matrix_multiple_prf()
prior.upper_lim_map()
print('fitting '+ str(prior.nsrc)+' sources \n')
print('using ' + str(prior.snpix) + ' pixels')
from xidplus.numpyro_fit import LOFAR150
fit=LOFAR150.LOFAR_150(prior,num_samples=1000,num_warmup=1000)
samples = fit.get_samples()['src_f']
posterior = xidplus.posterior_numpyro(fit,[prior])
if os.path.exists('data/xidplus_results/injected_sources/xidplus_run_{}'.format(n))==True:()
else:
os.mkdir('data/xidplus_results/injected_sources/xidplus_run_{}'.format(n))
xidplus.save([prior],posterior,'data/xidplus_results/injected_sources/xidplus_run_{}/injected_sources_{}.pkl'.format(n,n))
LOFAR_cat=cat.create_LOFAR_cat(posterior,prior)
LOFAR_cat = Table.read(LOFAR_cat)
with serialize_method_as(LOFAR_cat, None):
registry.write(LOFAR_cat,'data/xidplus_results/injected_sources/xidplus_run_{}/injected_sources_cat_{}.fits'.format(n,n),format='fits',overwrite=True)
prior160.set_prf(pacs160_psf[1].data[centre160-radius160:centre160+radius160+1,centre160-radius160:centre160+radius160+1]/1000.0,
pind160,pind160)
prior100.get_pointing_matrix()
prior160.get_pointing_matrix()
from xidplus.stan_fit import PACS
fit=PACS.all_bands(prior100,prior160,iter=1000)
posterior=xidplus.posterior_stan(fit,[prior100,prior160])
priors = [prior100,prior160]
import xidplus.catalogue as cat
PACS_cat=cat.create_PACS_cat(posterior,priors[0],priors[1])
PACS_cat = Table.read(PACS_cat)
mask = [PACS_cat['help_id'][i] in ids for i in range(len(PACS_cat))]
PACS_cat = PACS_cat[mask]
if os.path.exists('data/fir/PACS/v2/xidplus_run_{}'.format(taskid))==True:()
else:
os.mkdir('data/fir/PACS/v2/xidplus_run_{}'.format(taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
xidplus.save([prior100,prior160],posterior,'data/fir/PACS/v2/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.pkl'.format(taskid,taskid))
with serialize_method_as(PACS_cat, None):
registry.write(PACS_cat,'data/fir/PACS/v2/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),format='fits',overwrite=True)
#Table.write(PACS_cat,'data/fir/PACS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),format='fits',overwrite=True)
priors = [prior250, prior350, prior500]
import xidplus.catalogue as cat
SPIRE_cat = cat.create_SPIRE_cat(posterior, priors[0], priors[1], priors[2])
SPIRE_cat = Table.read(SPIRE_cat)
mask = ['ILTJ' in SPIRE_cat['HELP_ID'][i] for i in range(len(SPIRE_cat))]
SPIRE_cat = SPIRE_cat[mask]
'''mask = SPIRE_cat['HELP_ID']=='lofar'
mask_lofar = lofar['ra']==ra
mask_pcat = prior_cat['help_id']=='lofar'
SPIRE_cat = SPIRE_cat[mask]
SPIRE_cat.add_columns([prior_cat['f_mips_24'][mask_pcat],prior_cat['flag_optnir_det'][mask_pcat]])
lofar_fir = hstack([lofar[mask_lofar],SPIRE_cat])
XID_rerun_col = Column(data=XID_rerun,name='XID_rerun',dtype=bool)
source_type_col = Column(data=source_type,name='source_type',dtype=str)
lofar_fir.add_columns([XID_rerun_col,source_type_col])'''
if os.path.exists('data/fir_v2/xidplus_run_{}'.format(taskid)) == True: ()
else:
os.mkdir('data/fir_v2/xidplus_run_{}'.format(taskid))
Table.write(
SPIRE_cat,
'data/fir_v2/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(
taskid, taskid),
overwrite=True)
xidplus.save(
[prior250, prior350, prior500], posterior,
'data/fir_v2/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.pkl'.format(
taskid, taskid))
prior250.upper_lim_map()
prior350.upper_lim_map()
prior500.upper_lim_map()
from xidplus.stan_fit import SPIRE
fit=SPIRE.all_bands(prior250,prior350,prior500,iter=1000)
posterior=xidplus.posterior_stan(fit,[prior250,prior350,prior500])
priors = [prior250,prior350,prior500]
import xidplus.catalogue as cat
SPIRE_cat=cat.create_SPIRE_cat(posterior,priors[0],priors[1],priors[2])
SPIRE_cat = Table.read(SPIRE_cat)
mask = [SPIRE_cat['HELP_ID'][i] in ids for i in range(len(SPIRE_cat))]
SPIRE_cat = SPIRE_cat[mask]
if os.path.exists('data/fir/SPIRE_no_lofar/xidplus_run_{}'.format(taskid))==True:()
else:
os.mkdir('data/fir/SPIRE_no_lofar/xidplus_run_{}'.format(taskid))
xidplus.save([prior250,prior350,prior500],posterior,'data/fir/SPIRE_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}_rerun'.format(taskid,taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
with serialize_method_as(SPIRE_cat, None):
registry.write(SPIRE_cat,'data/fir/SPIRE_no_lofar/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),format='fits',overwrite=True)
#Table.write(SPIRE_cat,'data/fir/SPIRE/xidplus_run_{}/lofar_xidplus_fir_{}.fits'.format(taskid,taskid),overwrite=True)
time1 = time.time()
print('total time taken = {}'.format(time1-time0))
else:
prfs.append(psf_dirty[0])
pinds.append(np.arange(0,prfs[m].shape[0],1))
prior.set_prfs(prfs,pinds,pinds)
prior.get_pointing_matrix_multiple_prf()
prior.upper_lim_map()
print('fitting '+ str(prior.nsrc)+' sources \n')
print('using ' + str(prior.snpix) + ' pixels')
from xidplus.numpyro_fit import LOFAR150
fit=LOFAR150.LOFAR_150(prior,num_samples=1000,num_warmup=1000)
samples = fit.get_samples()['src_f']
posterior = xidplus.posterior_numpyro(fit,[prior])
if os.path.exists('data/xidplus_results/random_positions/xidplus_run_{}'.format(n))==True:()
else:
os.mkdir('data/xidplus_results/random_positions/xidplus_run_{}'.format(n))
xidplus.save([prior],posterior,'data/xidplus_results/random_positions/xidplus_run_{}/random_positions_{}.pkl'.format(n,n))
LOFAR_cat=cat.create_LOFAR_cat(posterior,prior)
LOFAR_cat = Table.read(LOFAR_cat)
with serialize_method_as(LOFAR_cat, None):
registry.write(LOFAR_cat,'data/xidplus_results/random_positions/xidplus_run_{}/random_positions_cat_{}.fits'.format(n,n),format='fits',overwrite=True)
posterior = xidplus.posterior_stan(fit, [prior250])
priors = [prior250]
import xidplus.catalogue as cat
MIPS_cat = cat.create_MIPS_cat(posterior, priors[0], 0)
MIPS_cat = Table.read(MIPS_cat)
mask = [MIPS_cat['help_id'][i] in ids for i in range(len(MIPS_cat))]
MIPS_cat = MIPS_cat[mask]
if os.path.exists('data/fir/v0_7/MIPS/xidplus_run_{}'.format(taskid)) == True:
()
else:
os.mkdir('data/fir/v0_7/MIPS/xidplus_run_{}'.format(taskid))
#the next couple of lines are an alternative way to save astropy table since the Table.write method is currently broken
with serialize_method_as(MIPS_cat, None):
registry.write(
MIPS_cat,
'data/fir/v0_7/MIPS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.
format(taskid, taskid),
format='fits',
overwrite=True)
#Table.write(MIPS_cat,'data/fir/MIPS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.fits'.format(taskid,taskid),overwrite=True)
xidplus.save(
[prior250], posterior,
'data/fir/v0_7/MIPS/xidplus_run_{}/lofar_xidplus_fir_{}_rerun.pkl'.format(
taskid, taskid))
