oldArray = data.field(16) # Copy of the old amplitude data array
oldCol = data.columns[16].copy() # Copy of the old corresponding header
newFormat = fi.column._ColumnFormat(
str(newSize) + 'E') # Definition of the new data array format
newDim = '(' + str(bin) + ',' + str(chan) + ',' + str(pol) + ',' + str(
newSamples) + ')' # Definition of the new data array definition
newArray = np.reshape(
oldArray,
(newBlocks, newSamples, pol, chan, bin)) # Resizing of the data array
newCol = fi.Column(name=oldCol.name,
format=newFormat,
unit=oldCol.unit,
dim=newDim,
array=newArray) # Creation of the new field
colList.append(newCol) # Adding to the new field list
# DEFINITION OF THE NEW FITS
colDefs = fi.ColDefs(colList) # Creation of the new fields object
tbhdu = fi.BinTableHDU.from_columns(
colDefs, header=head) # Creation of the new data table object
prihdu = fi.PrimaryHDU(
header=headObs
) # Creation of the new observation header (exactly the same that the old fits file)
hdulist = fi.HDUList([prihdu, tbhdu]) # Creation of the new HDU object
hdulist.writeto(args.newFileName, output_verify='exception'
) # Writing the new HDU object on the new fits file
hdulist.close()
def extract_spectrum_viasourcemodelcube(
datacube,
sourceweights,
wavelengths,
specname='tdose_extract_spectra_extractedspec.fits',
noisecube=None,
spec1Dmethod='sum',
sourcecube_hdr='None',
verbose=True):
"""
Extracting a spectrum from a data cube given a source model (cube) to be used as 'extraction cube'
--- INPUT ---
datacube Datacube to extract spectra from
sourceweights Weights from source model to use as "extraction cube". The weights should contain the
fractional flux belonging to the source in each pixel
wavelengths Wavelength vector to use for extracted 1D spectrum.
specname Name of spectrum to generate
noisecube Cube with uncertainties (sqrt(variance)) of data cube to be used in extraction
spec1Dmethod Method used to extract 1D spectrum from source cube with
sourcecube_hdr If not 'None' provide a fits header for the source cube and it ill be appended to the
output fits file.
verbose Toggle verbosity
--- EXAMPLE OF USE ---
"""
if verbose: print ' - Checking shape of data and source model cubes'
if datacube.shape != sourceweights.shape:
sys.exit(' ---> Shape of datacube (' + str(datacube.shape) +
') and source weights (' + sourceweights.shape +
') do not match.')
else:
if verbose: print ' dimensions match; proceeding with extraction '
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose:
print ' - Applying weights to "datacube" to obtain source cube '
sourcecube = datacube * sourceweights
if noisecube is not None:
if verbose: print ' - Using "noisecube" for error propagation '
datanoise = noisecube
else:
if verbose: print ' - No "noisecube" provided. Setting all errors to 1'
datanoise = np.ones(datacube.shape)
if verbose:
print ' - Assuming uncertainty on source weights equals the datanoise when propgating errors'
sourceweights_err = datanoise
sourcecube_err = sourcecube * np.sqrt(
(datanoise / datacube)**2 + (sourceweights_err / sourceweights)**2)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose: print ' - Generating 1D spectrum from source cube via:'
spec_wave = wavelengths
maskinvalid = np.ma.masked_invalid(sourcecube * sourcecube_err).mask
if spec1Dmethod == 'sum':
if verbose: print ' Simple summation of fluxes in sourcecube.'
spec_flux = np.sum(np.sum(np.ma.array(sourcecube, mask=maskinvalid),
axis=1),
axis=1).filled()
if verbose: print ' Errors are propagated as sum of squares.'
spec_err = np.sqrt(
np.sum(np.sum(np.ma.array(sourcecube_err, mask=maskinvalid)**2,
axis=1),
axis=1)).filled()
elif spec1Dmethod == 'sum_SNweight':
pdb.set_trace()
else:
sys.exit(' ---> The chosen spec1Dmethod (' + str(spec1Dmethod) +
') is invalid')
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose:
print ' - Saving extracted 1D spectrum and source cube to \n ' + specname
mainHDU = pyfits.PrimaryHDU() # primary HDU
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
c1 = pyfits.Column(name='wave',
format='D',
unit='ANGSTROMS',
array=spec_wave)
c2 = pyfits.Column(name='flux', format='D', unit='', array=spec_flux)
c3 = pyfits.Column(name='fluxerror', format='D', unit='', array=spec_err)
coldefs = pyfits.ColDefs([c1, c2, c3])
th = pyfits.new_table(coldefs) # creating default header
# writing hdrkeys:'---KEY--', '----------------MAX LENGTH COMMENT-------------'
th.header.append(('EXTNAME ', 'SPEC1D', 'cube containing source'),
end=True)
th.header.append(
('SPECMETH', spec1Dmethod, 'Method used for spectral extraction'),
end=True)
head = th.header
tbHDU = pyfits.new_table(coldefs, header=head)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if sourcecube_hdr != 'None':
sourceHDU = pyfits.ImageHDU(
sourcecube) # default HDU with default minimal header
for hdrkey in sourcecube_hdr.keys():
if not hdrkey in sourceHDU.header.keys():
sourceHDU.header.append((hdrkey, sourcecube_hdr[hdrkey],
sourcecube_hdr.comments[hdrkey]),
end=True)
sourceHDU.header.append(
('EXTNAME ', 'SOURCECUBE', 'cube containing source'), end=True)
hdulist = pyfits.HDUList([mainHDU, tbHDU, sourceHDU])
else:
hdulist = pyfits.HDUList([mainHDU, tbHDU])
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
hdulist.writeto(specname, clobber=True)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
return sourcecube, sourcecube_err, spec_wave, spec_flux, spec_err
def write_array(arr,
filename,
names=(),
units=(),
header_dict={},
ext='new',
close=True):
"""
Write or add an array to a FITS file.
If 'filename' refers to an existing file, the list of arrays will be added
(ext='new') to the HDUlist or replace an existing HDU (ext=integer). Else,
a new file will be created.
Names and units should be given as a list of strings, in the same order as
the list of arrays.
A header_dictionary can be given, it is used to update an existing header
or create a new one if the extension is new.
Instead of writing the file, you can give a hdulist and append to it.
Supply a HDUList for 'filename', and set close=False
"""
if isinstance(filename, str) and not os.path.isfile(filename):
primary = np.array([[0]])
hdulist = pyfits.HDUList([pyfits.PrimaryHDU(primary)])
hdulist.writeto(filename)
if isinstance(filename, str):
hdulist = pyfits.open(filename, mode='update')
else:
hdulist = filename
#-- create the table HDU
cols = []
for i, name in enumerate(names):
format = arr[i].dtype.str.lower().replace('|', '').replace(
's', 'a').replace('>', '')
format = format.replace('b1', 'L').replace('<', '')
if format == 'f8':
format = 'D'
if isinstance(units, dict):
unit = name in units and units[name] or 'NA'
elif len(units) > i:
unit = units[i]
else:
unit = 'NA'
cols.append(
pyfits.Column(name=name, format=format, array=arr[i], unit=unit))
tbhdu = pyfits.new_table(pyfits.ColDefs(cols))
# put it in the right place
if ext == 'new' or ext == len(hdulist):
hdulist.append(tbhdu)
ext = -1
else:
hdulist[ext] = tbhdu
#-- take care of the header:
if len(header_dict):
for key in header_dict:
hdulist[ext].header.update(key, header_dict[key])
if close:
hdulist.close()
else:
return hdulist
#}
def extract_pdf(model,
ml_dataset,
z_phot=None,
randomized_datasets=None,
pdf_grid_size=100,
pdf_grid_min=None,
pdf_grid_max=None,
gmm_components=2,
out_file_name=None,
skip_gmm=False):
"""
Fit a Gaussian Mixture Model (GMM) to the redshift distribution and return an array where each element represents the PDF of a sample.
The PDF for each sample is a tuple that contains the following data
- The sample's ID as given in the input data
- The spectroscopic redshift from the input data
- The photometric redshift as predicted by AdaBoost
- The redshift predicted by each estimator in AdaBoost
- The grid of redshifts for which the value of the PDF is given
- The PDF resulting from the GMM
- The mean mu of the GMM
- The variance sigma of the GMM
- The weights assigned to each model in the GMM
If `out_file_name` is specified, then the PDFs are also written to a file
Parameters
----------
model : a :class:`PrimalCore.models.base.BaseModel` based model
ml_dataset : :class:`PrimalCore.homogeneous_table.dataset.MLDataSet` dataset
z_phot : 1dim array
predicted z_phot, if None is evalauted from model and ml_dataset
pdf_grid_size : int (Optional)
the size of the grid to evaluate the pdf
pdf_grid_min : float (Optional)
max z value for pdf bin
pdf_grid_max : float (Optional)
max z value for pdf bin
gmm_components : int (Optional) default=5
the number of components in teh GMM model to test
out_file_name : basestring
output file name
Returns
-------
pdf : ndarray
Array with the PDF for each sample
"""
if hasattr(model.clf, 'estimators_') == False:
return None
if z_phot is None:
z_phot = model.clf.predict(ml_dataset.features)
if randomized_datasets is None:
trials = len(model.clf.estimators_)
pred_z_phot = model.eval_estimators_predictions(ml_dataset.features)
else:
trials = randomized_datasets.shape[0]
pred_z_phot = np.zeros((ml_dataset.features_N_rows, trials))
print('trials', trials, randomized_datasets.shape, pred_z_phot.shape)
for trial in range(trials):
pred_z_phot[:,
trial] = model.clf.predict(randomized_datasets[trial])
print('prediction on random done')
c1 = pf.Column(name='original_row_ID',
format='J',
array=ml_dataset.features_original_entry_ID)
if ml_dataset._target_array is not None:
z_spec = ml_dataset.target_array
else:
z_spec = np.ones(ml_dataset.features_N_rows) * -1
c2 = pf.Column(name='z_spec', format='D', array=z_spec)
c3 = pf.Column(name='z_phot', format='D', array=z_phot)
c4 = pf.Column(
name='z_phot_values',
format='%dD' % (trials),
)
c5 = pf.Column(
name='z_phot_pdf_grid',
format='%dD' % (pdf_grid_size),
)
c6 = pf.Column(
name='z_phot_pdf',
format='%dD' % (pdf_grid_size),
)
c7 = pf.Column(
name='z_gmm_mu',
format='%dD' % (gmm_components),
)
c8 = pf.Column(
name='z_gmm_sig',
format='%dD' % (gmm_components),
)
c9 = pf.Column(
name='z_gmm_w',
format='%dD' % (gmm_components),
)
coldefs = pf.ColDefs([c1, c2, c3, c4, c5, c6, c7, c8, c9])
tbhdu = pf.BinTableHDU.from_columns(coldefs)
#
for trial in range(trials):
tbhdu.data['z_phot_values'][:, trial] = pred_z_phot[:, trial]
if skip_gmm is False:
for entry in range(pred_z_phot.shape[0]):
z_grid, gmm_pdf, mu, sig, w = eval_pdf_gmm(
z_values=pred_z_phot[entry],
grid_size=pdf_grid_size,
grid_max=pdf_grid_max,
grid_min=pdf_grid_min,
n_components=gmm_components)
n_components = len(mu)
tbhdu.data['z_phot_pdf_grid'][entry] = z_grid
tbhdu.data['z_phot_pdf'][entry] = gmm_pdf
tbhdu.data['z_gmm_mu'][entry][:n_components] = mu
tbhdu.data['z_gmm_sig'][entry][:n_components] = sig
tbhdu.data['z_gmm_w'][entry][:n_components] = w
if out_file_name is not None:
header_tuple_list = [('cat_file', ml_dataset.catalog_file,
'catalog file')]
write_data(out_file_name,
tbhdu.data,
extra_header_tuple_list=header_tuple_list)
return np.array(tbhdu.data)
m2_table, c1_table, c2_table, Tree,
scale_x, scale_y, shift_x, shift_y)
elif (cal_type == "bin"):
cals = bias_2D_bin_query(snr[i], resolution[i], table_data)
elif (cal_type == "function"):
cals = bias_2D_function_query(snr[i], resolution[i], function,
function, m1_params, m2_params)
m1.append(cals[0])
m2.append(cals[1])
c1.append(cals[2])
c2.append(cals[3])
m1 = array(m1)
m2 = array(m2)
c1 = array(c1)
c2 = array(c2)
nc1 = pyfits.Column(name='m1_' + col_name, format='D', array=m1)
nc2 = pyfits.Column(name='m2_' + col_name, format='D', array=m2)
nc3 = pyfits.Column(name='c1_' + col_name, format='D', array=c1)
nc4 = pyfits.Column(name='c2_' + col_name, format='D', array=c2)
old_cols = fits_data[1].columns
cols = (nc1, nc2, nc3, nc4)
new_cols = pyfits.ColDefs(cols)
new_hdu = pyfits.new_table(old_cols + new_cols)
new_hdu.writeto(outfile, clobber=True)
else:
print "Chosen input or calibration file does not exist."
def savefitstable(self, name='', header='', format={}, labels=1, overwrite=1): # FITS TABLE
# if name then name, else self.name
name = name or recapfile(self.name, 'fits')
name = capfile(name, 'fits') # IF WAS name (PASSED IN) NEED TO capfile
if (not overwrite) and os.path.exists(name):
print(name, 'ALREADY EXISTS, AND YOU TOLD ME NOT TO OVERWRITE IT')
else:
units = self.units
header = header or self.header # if header then header, else self.header
if type(labels) == list:
self.labels = labels
labels = labels and self.labels # if labels then self.labels, else 0
collist = []
for label in self.labels:
a = np.array(self.get(label))
#if not pyfitsusesnumpy:
# a = numarray.array(a)
if label in list(units.keys()):
col = pyfits.Column(name=label, format=format.get(
label, 'E'), unit=units[label], array=a)
else:
col = pyfits.Column(
name=label, format=format.get(label, 'E'), array=a)
collist.append(col)
cols = pyfits.ColDefs(collist)
tbhdu = pyfits.new_table(cols)
if not self.descriptions:
delfile(name)
tbhdu.writeto(name)
else:
hdu = pyfits.PrimaryHDU()
hdulist = pyfits.HDUList(hdu)
hdulist.append(tbhdu)
prihdr = hdulist[1].header
descriptions = self.descriptions
for ilabel in range(len(labels)):
label = labels[ilabel]
if label in list(descriptions.keys()):
description = descriptions[label]
if len(description) < 48:
description1 = description
description2 = ''
else:
i = description[:45].rfind(' ')
description1 = description[:i] + '...'
description2 = '...' + description[i + 1:]
prihdr.update('TTYPE%d' %
(ilabel + 1), label, description1)
if description2:
prihdr.update('TFORM%d' % (ilabel + 1),
format.get(label, 'E'), description2)
for inote in range(len(self.notes)):
words = self.notes[inote].split('\n')
for iword in range(len(words)):
word = words[iword]
if word:
if iword == 0:
prihdr.add_comment(
'(%d) %s' % (inote + 1, word))
else:
prihdr.add_comment(' %s' % word)
# prihdr.add_blank(word)
headlines = header.split('\n')
for headline in headlines:
if headline:
key, value = headline.split('\t')
prihdr.update(key, value)
hdulist.writeto(name)
plt.ylim((0., 4.))
plt.show()
# Plot radius vs. flux for Sersic fits
log_flux = np.log10(flux[viable_sersic, 0])
log_rad = np.log10(hlr[viable_sersic, 0])
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(log_flux, log_rad, 'bo', linestyle='None')
plt.xlabel('Log sersic flux')
plt.ylabel('Log sersic radius')
plt.xlim((-1., 4.))
plt.ylim((-2.5, 1.))
plt.show()
# Stick them together into a single FITS table. First just make a table with the new stuff.
tbhdu = pyfits.new_table(
pyfits.ColDefs([
pyfits.Column(name='use_bulgefit', format='J', array=use_bulgefit),
pyfits.Column(name='viable_sersic', format='J', array=viable_sersic),
pyfits.Column(name='hlr', format='3D', array=hlr),
pyfits.Column(name='flux', format='4D', array=flux)
]))
# Then merge them.
new_table = dat.columns + tbhdu.columns
hdu = pyfits.new_table(new_table)
# Output to file.
out_file = os.path.join(data_dir, param_fit_file)
print "Writing to file ", out_file
hdu.writeto(out_file, clobber=True)
def fix_grid(grid):
hdulist = pyfits.open(grid, mode='update')
names = hdulist[1].columns.names
for i, name in enumerate(names):
if name.lower() in ['teff', 'logg', 'ebv', 'labs', 'vrad', 'rv', 'z']:
names[i] = name.lower()
cols = [
pyfits.Column(name=name, format='E', array=hdulist[1].data.field(name))
for name in names
]
N = len(hdulist[1].data)
keys = [key.lower() for key in hdulist[1].header.keys()]
if not 'z' in names:
z = hdulist[1].header['z']
logger.info('Adding metallicity from header {}'.format(z))
cols.append(pyfits.Column(name='z', format='E', array=np.ones(N) * z))
else:
logger.info("Metallicity already in there")
if not 'vrad' in names:
vrad = 0.
logger.info('Adding radial velocity {}'.format(vrad))
cols.append(
pyfits.Column(name='vrad', format='E', array=np.ones(N) * vrad))
else:
logger.info("Radial velocity already in there")
fix_rv = False
if not 'rv' in names:
if 'rv' in keys:
rv = hdulist[1].header['Rv']
logger.info("Adding interstellar Rv from header {}".format(rv))
else:
rv = 3.1
logger.info("Adding default interstellar Rv {}".format(rv))
cols.append(pyfits.Column(name='rv', format='E',
array=np.ones(N) * rv))
elif not hdulist[1].header['Rv'] == hdulist[1].data.field('rv')[0]:
rv = hdulist[1].header['Rv']
fix_rv = rv
logger.info('Correcting interstellar Rv with {}'.format(rv))
else:
logger.info("Interstellar Rv already in there")
table = pyfits.new_table(pyfits.ColDefs(cols))
if fix_rv:
table.data.field('rv')[:] = rv
fake_keys = [key.lower() for key in table.header.keys()]
fake_keys.append(
'use_scratch'
) # Don't know why this is nessessary but it doesn't work otherwise (JV 23.7.13) !!!
for key in hdulist[1].header.keys():
if not key.lower() in fake_keys:
if len(key) > 8:
key = 'HIERARCH ' + key
table.header.update(key, hdulist[1].header[key])
hdulist[1] = table
print "Axis:"
for name in hdulist[1].columns.names:
if name.islower() and not name == 'labs':
ax = np.unique(hdulist[1].data.field(name))
print name, len(ax), min(ax), max(ax)
teffs = hdulist[1].data.field('teff')
loggs = hdulist[1].data.field('logg')
ebvs = hdulist[1].data.field('ebv')
#for logg in np.unique(loggs):
#keep = loggs==logg
#plt.figure()
#plt.title(logg)
#plt.plot(teffs[keep],ebvs[keep],'ko',ms=2)
keep = hdulist[1].data.field('teff') > 0
logger.info('Removing {}/{} false entries'.format(sum(-keep), len(keep)))
#print len(ff[1].data),sum(keep)
hdulist[1].data = hdulist[1].data[keep]
hdulist.close()
else:
start[i] = badtimeend[j]
else:
break
start = delete(start, badlist)
stop = delete(stop, badlist)
errbar = 0.5 * (stop - start)
center = array(start + errbar) #-starttime
#array=0.*start+10.
array = array(0. * start + max(counts) / 2)
#plotxy(array,center,symbol=1,line=None,errx=errbar,setup=0)
#closeplot()
print sum(stop - start)
col1 = pyfits.Column(name="START", format='D', unit='s', array=start)
col2 = pyfits.Column(name="STOP", format='D', unit='s', array=stop)
cols = pyfits.ColDefs([col1, col2])
tbhdu = pyfits.new_table(cols)
hdulist.append(tbhdu)
hdulist[2].header = hdulist[1].header
#print hdulist[2].header['ONTIME'],hdulist[2].header['TSTART'],hdulist[2].header['TSTOP']
hdulist[2].header['ONTIME'] = sum(stop - start)
hdulist[2].header['TSTART'] = start[0]
hdulist[2].header['TSTOP'] = stop[len(stop) - 1]
#print hdulist[2].header['ONTIME'],hdulist[2].header['TSTART'],hdulist[2].header['TSTOP']
hdulist.remove(hdulist[1])
hdulist.writeto('newgti.fits')
hdulist.close()
#plotxy(counts,time,device="gti.ps/PS")
#plotxy(array,center,symbol=1,line=None,errx=errbar,setup=0)
#closeplot()
def calc_integrated_grid(threads=1,
ebvs=None,
law='fitzpatrick2004',
Rv=3.1,
units='Flambda',
responses=None,
update=False,
add_spectrophotometry=False,
**kwargs):
"""
Integrate an entire SED grid over all passbands and save to a FITS file.
The output file can be used to fit SEDs more efficiently, since integration
over the passbands has already been carried out.
WARNING: this function can take a loooooong time to compute!
Extra keywords can be used to specify the grid.
@param threads: number of threads
@type threads; integer, 'max', 'half' or 'safe'
@param ebvs: reddening parameters to include
@type ebvs: numpy array
@param law: interstellar reddening law to use
@type law: string (valid law name, see C{reddening.py})
@param Rv: Rv value for reddening law
@type Rv: float
@param units: choose to work in 'Flambda' or 'Fnu'
@type units: str, one of 'Flambda','Fnu'
@param responses: respons curves to add (if None, add all)
@type responses: list of strings
@param update: if true append to existing FITS file, otherwise overwrite
possible existing file.
@type update: boolean
"""
if ebvs is None:
ebvs = np.r_[0:4.01:0.01]
#-- select number of threads
if threads == 'max':
threads = cpu_count()
elif threads == 'half':
threads = cpu_count() / 2
elif threads == 'safe':
threads = cpu_count() - 1
threads = int(threads)
if threads > len(ebvs):
threads = len(ebvs)
logger.info('Threads: %s' % (threads))
#-- set the parameters for the SED grid
model.set_defaults(**kwargs)
#-- get the dimensions of the grid: both the grid points, but also
# the wavelength range
teffs, loggs = model.get_grid_dimensions()
wave, flux = model.get_table(teff=teffs[0], logg=loggs[0])
#-- get the response functions covering the wavelength range of the models
# also get the information on those filters
responses = get_responses(responses=responses,\
add_spectrophotometry=add_spectrophotometry,wave=wave)
#-- definition of one process:
def do_ebv_process(ebvs, arr, responses):
logger.debug('EBV: %s-->%s (%d)' % (ebvs[0], ebvs[-1], len(ebvs)))
for ebv in ebvs:
flux_ = reddening.redden(flux,
wave=wave,
ebv=ebv,
rtype='flux',
law=law,
Rv=Rv)
#-- calculate synthetic fluxes
synflux = model.synthetic_flux(wave, flux_, responses, units=units)
arr.append([np.concatenate(([ebv], synflux))])
logger.debug("Finished EBV process (len(arr)=%d)" % (len(arr)))
#-- do the calculations
c0 = time.time()
output = np.zeros((len(teffs) * len(ebvs), 4 + len(responses)))
start = 0
logger.info('Total number of tables: %i' % (len(teffs)))
exceptions = 0
exceptions_logs = []
for i, (teff, logg) in enumerate(zip(teffs, loggs)):
if i > 0:
logger.info('%s %s %s %s: ET %d seconds' %
(teff, logg, i, len(teffs),
(time.time() - c0) / i * (len(teffs) - i)))
#-- get model SED and absolute luminosity
wave, flux = model.get_table(teff=teff, logg=logg)
Labs = model.luminosity(wave, flux)
#-- threaded calculation over all E(B-V)s
processes = []
manager = Manager()
arr = manager.list([])
all_processes = []
for j in range(threads):
all_processes.append(
Process(target=do_ebv_process,
args=(ebvs[j::threads], arr, responses)))
all_processes[-1].start()
for p in all_processes:
p.join()
try:
#-- collect the results and add them to 'output'
arr = np.vstack([row for row in arr])
sa = np.argsort(arr[:, 0])
arr = arr[sa]
output[start:start + arr.shape[0], :3] = teff, logg, Labs
output[start:start + arr.shape[0], 3:] = arr
start += arr.shape[0]
except:
logger.warning('Exception in calculating Teff=%f, logg=%f' %
(teff, logg))
logger.debug('Exception: %s' % (sys.exc_info()[1]))
exceptions = exceptions + 1
exceptions_logs.append(sys.exc_info()[1])
#-- make FITS columns
gridfile = model.get_file()
if os.path.isfile(os.path.basename(gridfile)):
outfile = os.path.basename(gridfile)
else:
outfile = os.path.join(os.path.dirname(gridfile),
'i{0}'.format(os.path.basename(gridfile)))
outfile = 'i{0}'.format(os.path.basename(gridfile))
outfile = os.path.splitext(outfile)
outfile = outfile[0] + '_law{0}_Rv{1:.2f}'.format(law, Rv) + outfile[1]
logger.info('Precaution: making original grid backup at {0}.backup'.format(
outfile))
if os.path.isfile(outfile):
shutil.copy(outfile, outfile + '.backup')
output = output.T
if not update or not os.path.isfile(outfile):
cols = [
pyfits.Column(name='teff', format='E', array=output[0]),
pyfits.Column(name='logg', format='E', array=output[1]),
pyfits.Column(name='ebv', format='E', array=output[3]),
pyfits.Column(name='Labs', format='E', array=output[2])
]
for i, photband in enumerate(responses):
cols.append(
pyfits.Column(name=photband, format='E', array=output[4 + i]))
#-- make FITS columns but copy the existing ones
else:
hdulist = pyfits.open(outfile, mode='update')
names = hdulist[1].columns.names
cols = [
pyfits.Column(name=name,
format='E',
array=hdulist[1].data.field(name)) for name in names
]
for i, photband in enumerate(responses):
cols.append(
pyfits.Column(name=photband, format='E', array=output[4 + i]))
#-- make FITS extension and write grid/reddening specifications to header
table = pyfits.new_table(pyfits.ColDefs(cols))
table.header.update('gridfile', os.path.basename(gridfile))
for key in sorted(model.defaults.keys()):
key_ = (len(key) > 8) and 'HIERARCH ' + key or key
table.header.update(key_, model.defaults[key])
for key in sorted(kwargs.keys()):
key_ = (len(key) > 8) and 'HIERARCH ' + key or key
table.header.update(key_, kwargs[key])
table.header.update('FLUXTYPE', units)
table.header.update('REDLAW', law, 'interstellar reddening law')
table.header.update('RV', Rv, 'interstellar reddening parameter')
#-- make/update complete FITS file
if not update or not os.path.isfile(outfile):
if os.path.isfile(outfile):
os.remove(outfile)
logger.warning('Removed existing file: %s' % (outfile))
hdulist = pyfits.HDUList([])
hdulist.append(pyfits.PrimaryHDU(np.array([[0, 0]])))
hdulist.append(table)
hdulist.writeto(outfile)
logger.info("Written output to %s" % (outfile))
else:
hdulist[1] = table
hdulist.flush()
hdulist.close()
logger.info("Appended output to %s" % (outfile))
logger.warning('Encountered %s exceptions!' % (exceptions))
for i in exceptions_logs:
print 'ERROR'
print i
def update_grid(gridfile, responses, threads=10):
"""
Add passbands to an existing grid.
"""
shutil.copy(gridfile, gridfile + '.backup')
hdulist = pyfits.open(gridfile, mode='update')
existing_responses = set(list(hdulist[1].columns.names))
responses = sorted(list(set(responses) - existing_responses))
if not len(responses):
hdulist.close()
print "No new responses to do"
return None
law = hdulist[1].header['REDLAW']
units = hdulist[1].header['FLUXTYPE']
teffs = hdulist[1].data.field('teff')
loggs = hdulist[1].data.field('logg')
ebvs = hdulist[1].data.field('ebv')
zs = hdulist[1].data.field('z')
rvs = hdulist[1].data.field('rv')
vrads = hdulist[1].data.field('vrad')
names = hdulist[1].columns.names
N = len(teffs)
index = np.arange(N)
output = np.zeros((len(responses), len(teffs)))
print N
#--- PARALLEL PROCESS
def do_process(teffs, loggs, ebvs, zs, rvs, index, arr):
output = np.zeros((len(responses) + 1, len(teffs)))
c0 = time.time()
N = len(teffs)
for i, (teff, logg, ebv, z, rv,
ind) in enumerate(zip(teffs, loggs, ebvs, zs, rvs, index)):
if i % 100 == 0:
dt = time.time() - c0
print "ETA", index[0], (N - i) / 100. * dt / 3600., 'hr'
c0 = time.time()
#-- get model SED and absolute luminosity
model.set_defaults(z=z)
wave, flux = model.get_table(teff, logg)
Labs = model.luminosity(wave, flux)
flux_ = reddening.redden(flux,
wave=wave,
ebv=ebv,
rtype='flux',
law=law,
Rv=rv)
#-- calculate synthetic fluxes
output[0, i] = ind
output[1:, i] = model.synthetic_flux(wave,
flux_,
responses,
units=units)
arr.append(output)
#--- PARALLEL PROCESS
c0 = time.time()
manager = Manager()
arr = manager.list([])
all_processes = []
for j in range(threads):
all_processes.append(Process(target=do_process,args=(teffs[j::threads],\
loggs[j::threads],\
ebvs[j::threads],\
zs[j::threads],\
rvs[j::threads],\
index[j::threads],arr)))
all_processes[-1].start()
for p in all_processes:
p.join()
output = np.hstack([res for res in arr])
del arr
sa = np.argsort(output[0])
output = output[:, sa][1:]
ascii.write_array(np.rec.fromarrays(output, names=responses),
'test.temp',
header=True)
#-- copy old columns and append new ones
cols = []
for i, photband in enumerate(responses):
cols.append(pyfits.Column(name=photband, format='E', array=output[i]))
#-- create new table
table = pyfits.new_table(pyfits.ColDefs(cols))
table = pyfits.new_table(hdulist[1].columns + table.columns,
header=hdulist[1].header)
hdulist[1] = table
hdulist.close()
def calc_limbdark_grid(responses=None,vrads=[0],ebvs=[0],zs=[0.],\
ld_law='claret',fitmethod='equidist_r_leastsq',
outfile=None,force=False,**kwargs):
"""
Calculate a grid of limb-darkening coefficients.
You need to specify a list of response curves, and a grid of radial velocity
(C{vrads}), metallicity (C{z}) and reddening (C{ebvs}) points. You can
choose which C{law} to fit and with which C{fitmethod}. Extra kwargs specify
the properties of the atmosphere grid to be used.
If you give a gridfile that already exists, the current file is simply
updated with the new passbands, i.e. all overlapping response curves will not
be recomputed. Unless you set C{force=True}, in which case previous calculations
will be overwritten. You'd probably only want to update or overwrite existing
files if you use the same C{vrads}, C{ebvs}, C{zs} etc...
The generated FITS file has the following structure:
1. The primary HDU is empty. The primary header contains only the fit
routine (FIT), LD law (LAW) and used grid (GRID)
2. Each table extension has the name of the photometric passband (e.g.
"GENEVA.V". Each record in the table extension has the following columns:
Teff, logg, ebv, vrad, z (the grid points) and a1, a2, a3, a4 (the limb
darkening coefficients) and Imu1 (the intensity in the center of the disk)
and SRS, dint (fit evaluation parameters, see L{ivs.sed.limbdark}).
Although the system and filter can be derived from the extension name,
there are also separate entries in the header for "SYSTEM" and "FILTER".
Example usage:
>>> calc_limbdark_grid(['MOST.V','GENEVA'],vrads=[0],ebvs=[0.06],zs=[0,0],\
... law='claret',fitmethod='equidist_r_leastsq',outfile='HD261903.fits')
"""
#-- collect response curves from user input
photbands = get_responses(responses)
if outfile is None:
outfile = '{}_{}_{}.fits'.format(
kwargs.get('grid', limbdark.defaults['grid']), ld_law, fitmethod)
#-- add the possibility to update an existing file if it already exists.
if os.path.isfile(outfile):
hdulist = pyfits.open(outfile, mode='update')
existing_bands = [ext.header['extname'] for ext in hdulist[1:]]
else:
hdulist = pyfits.HDUList([])
hdulist.append(pyfits.PrimaryHDU(np.array([[0, 0]])))
existing_bands = []
#-- update the header with some information on the fitting parameters
hd = hdulist[0].header
hd.update('FIT', fitmethod, 'FIT ROUTINE')
hd.update('LAW', ld_law, 'FITTED LD LAW')
hd.update('GRID', kwargs.get('grid', limbdark.defaults['grid']), 'GRID')
#-- cycle through all the bands and compute the limb darkening coefficients
for i, photband in enumerate(photbands):
if photband in existing_bands and not force:
logger.info('BAND {} already exists: skipping'.format(photband))
continue
logger.info('Calculating photband {} ({}/{})'.format(
photband, i + 1, len(photbands)))
pars,coeffs,Imu1s = limbdark.fit_law_to_grid(photband,vrads=vrads,ebvs=ebvs,zs=zs,\
law=ld_law,fitmethod=fitmethod,**kwargs)
cols = []
cols.append(pyfits.Column(name='Teff', format='E', array=pars[:, 0]))
cols.append(pyfits.Column(name="logg", format='E', array=pars[:, 1]))
cols.append(pyfits.Column(name="ebv", format='E', array=pars[:, 2]))
cols.append(pyfits.Column(name="vrad", format='E', array=pars[:, 3]))
cols.append(pyfits.Column(name="z", format='E', array=pars[:, 4]))
for col in range(coeffs.shape[1]):
cols.append(
pyfits.Column(name='a{:d}'.format(col + 1),
format='E',
array=coeffs[:, col]))
cols.append(pyfits.Column(name='Imu1', format='E', array=Imu1s[:, 0]))
cols.append(pyfits.Column(name='SRS', format='E', array=Imu1s[:, 1]))
cols.append(pyfits.Column(name='dint', format='E', array=Imu1s[:, 2]))
newtable = pyfits.new_table(pyfits.ColDefs(cols))
newtable.header.update('EXTNAME', photband, "SYSTEM.FILTER")
newtable.header.update('SYSTEM',
photband.split('.')[0], 'PASSBAND SYSTEM')
newtable.header.update('FILTER',
photband.split('.')[1], 'PASSBAND FILTER')
if photband in existing_bands and force:
hdulist[hdulist.index_of(photband)] = newtable
logger.info("Forced overwrite of {}".format(photband))
else:
hdulist.append(newtable)
#-- clean up
if os.path.isfile(outfile):
hdulist.close()
else:
hdulist.writeto(outfile)
# Load the appropriate data from the parametric fit file.
dat = pyfits.getdata(os.path.join(data_dir, file))
# And set up the COSMOSCatalog
ccat = galsim.COSMOSCatalog(file, dir=data_dir, exclusion_level='none')
print 'Read in ', ccat.nobjects, ' objects'
# Compute the flux in the postage stamp
stamp_flux = []
for ind_n in range(ccat.nobjects):
if ind_n % 1000 == 0:
print ' Working on galaxy %d.' % ind_n
gal = ccat.makeGalaxy(index=ind_n, gal_type='real')
stamp_flux.append(gal.gal_image.array.sum())
stamp_flux = np.array(stamp_flux)
print len(stamp_flux[stamp_flux < 0]), ' negative flux stamps'
# Make a tbhdu for this data.
tbhdu = pyfits.new_table(
pyfits.ColDefs(
[pyfits.Column(name='stamp_flux', format='F', array=stamp_flux)]))
# Then merge with original table.
new_table = dat.columns + tbhdu.columns
hdu = pyfits.new_table(new_table)
# Output to file.
out_file = os.path.join(data_dir, file)
print "Writing to file ", out_file
hdu.writeto(out_file, clobber=True)
print ''
ndata =len(ut10)
#lat=s10['gyro_hid'].data['hybridlatitude']
#lon=s10['gyro_hid'].data['hybridlongitude']
my=d10['ch14_'].data['t']
mz=d10['ch15_'].data['t']
magang10=np.arctan2(mz,my)
#add 180 degrees, makes later fitting work well
magang10=magang10+np.pi
magang10[magang10>np.pi]=magang10[magang10>np.pi]-np.pi*2.0
d10.close()
# make the fits file and fill
cols=[]
cols.append(pyfits.Column(name='UT',format='Float32',array=ut10))
cols.append(pyfits.Column(name='AZ',format='Float32',array=magang10))
coldefinitions=pyfits.ColDefs(cols)
tablehdu=pyfits.new_table(coldefinitions)
tablehdu.name='10GHz'
tablehdu.writeto(magazfile10,clobber=True)
d15=pyfits.open('c:/cofe/flight_data/Level1/1.1/all_15GHz_data.fits')
ut15=d15['TIME'].data['UT']
ndata =len(ut15)
#lat=s10['gyro_hid'].data['hybridlatitude']
#lon=s10['gyro_hid'].data['hybridlongitude']
my=d15['ch14_'].data['t']
mz=d15['ch15_'].data['t']
magang15=np.arctan2(mz,my)
#add 180 degrees, makes later fitting work well
# sort in frequency space sinds = vs.argsort() # generate fits frame with sed in it primary_hdu = pyfits.PrimaryHDU(seds[sinds,:]) primary_hdu.header.update( 'units', 'ergs/s/cm^2/Hz' ) primary_hdu.header.update( 'has_seds', True ) primary_hdu.header.update( 'nfilters', 0 ) primary_hdu.header.update( 'nzfs', 0 ) # store meta data if sfh and met and imf: primary_hdu.header.update( 'has_meta', True ) primary_hdu.header.update( 'model', 'C09', comment='meta data' ) primary_hdu.header.update( 'met', met, comment='meta data' ) primary_hdu.header.update( 'imf', imf, comment='meta data' ) primary_hdu.header.update( 'sfh', sfh, comment='meta data' ) if sfh == 'Exponential': primary_hdu.header.update( 'tau', tau, comment='meta data' ) # store the list of frequencies in a table vs_hdu = pyfits.new_table(pyfits.ColDefs([pyfits.Column(name='vs', array=vs[sinds], format='D', unit='hertz')])) # and the list of ages + masses ages_hdu = pyfits.new_table(pyfits.ColDefs( [pyfits.Column(name='ages', array=ages, format='D', unit='years'), pyfits.Column(name='masses', array=masses, format='D', unit='m_sun')] )) ages_hdu.header.update( 'has_mass', True ) # make the fits file in memory hdulist = pyfits.HDUList( [primary_hdu,vs_hdu,ages_hdu] ) # and write it out hdulist.writeto( fileout, clobber=True )
dat = pyfits.getdata(os.path.join(data_dir, sn_file))
sn_ellip_gauss = dat['sn_ellip_gauss']
dat = pyfits.getdata(os.path.join(data_dir, mask_file))
min_mask_dist_pixels = dat['min_mask_dist_pixels']
average_mask_adjacent_pixel_count = dat['average_mask_adjacent_pixel_count']
peak_image_pixel_count = dat['peak_image_pixel_count']
# Stick them together into a single FITS table.
tbhdu = pyfits.new_table(
pyfits.ColDefs([
pyfits.Column(name='IDENT', format='J', array=ident),
pyfits.Column(name='dmag', format='D', array=dmag),
pyfits.Column(name='sn_ellip_gauss', format='D', array=sn_ellip_gauss),
pyfits.Column(name='min_mask_dist_pixels',
format='D',
array=min_mask_dist_pixels),
pyfits.Column(name='average_mask_adjacent_pixel_count',
format='D',
array=average_mask_adjacent_pixel_count),
pyfits.Column(name='peak_image_pixel_count',
format='D',
array=peak_image_pixel_count)
]))
# Output to file.
out_file = os.path.join(data_dir, out_file)
print "Writing to file ", out_file
tbhdu.writeto(out_file)
def write_pcm(sfname,
receiver_params,
valid_chans,
outname="pcm.fits",
fake=False):
#This method writes out the receiver parameters into a file that's readable/usable by PSRCHIVE.
#Most of it is getting the header/data formatting correct, so it's not very enlightening, but it is useful.
shdulist = pyfits.open(sfname)
head = shdulist[0].header
head['OBS_MODE'] = "PCM"
nchan = head['OBSNCHAN']
prihdu = pyfits.PrimaryHDU(header=head)
new_hdulist = pyfits.BinTableHDU(header=shdulist[1].header,
data=shdulist[1].data,
name=shdulist[1].name)
chans = np.linspace(head['OBSFREQ'] - head['OBSBW'] / 2.0,
head['OBSFREQ'] + head['OBSBW'] / 2.0,
nchan,
dtype='d')
weights = np.zeros(nchan)
weights[valid_chans] = 1
Gs = weights.reshape(nchan, 1)
expanded_receivers = np.zeros(nchan * 6).reshape(nchan, 6)
vchan = 0
for i in range(nchan):
if i in valid_chans:
expanded_receivers[i] = receiver_params[vchan, [5, 4, 2, 0, 3, 1]]
vchan += 1
params = np.append(Gs, expanded_receivers, axis=1)
if not fake:
params[params == 0] = np.NaN
ncovar = 28
ncov = ncovar * nchan
covars = np.ones(ncov) * 0.0005
write_params = np.concatenate(params)
chisqs = np.ones(nchan)
nfree = np.copy(weights)
chans = np.asarray([chans])
weights = np.asarray([weights])
write_params = np.asarray([write_params])
covars = np.asarray([covars])
chisqs = np.asarray([chisqs])
nfree = np.asarray([nfree])
freqcol = pyfits.Column(name="DAT_FREQ", format="512D", array=chans)
wtcol = pyfits.Column(name="DAT_WTS", format="512E", array=weights)
datcol = pyfits.Column(name="DATA", format="3584E", array=write_params)
covcol = pyfits.Column(name="COVAR", format="14336E", array=covars)
chicol = pyfits.Column(name="CHISQ", format="512E", array=chisqs)
freecol = pyfits.Column(name="NFREE", format="512J", array=nfree)
cols = pyfits.ColDefs([freqcol, wtcol, datcol, covcol, chicol, freecol])
feed_hdu = pyfits.BinTableHDU.from_columns(cols, name="FEEDPAR")
feed_hdu.header.comments[
'TTYPE1'] = '[MHz] Centre frequency for each channel'
feed_hdu.header.comments['TFORM1'] = 'NCHAN doubles'
feed_hdu.header.comments['TTYPE2'] = 'Weights for each channel'
feed_hdu.header.comments['TFORM2'] = 'NCHAN floats'
feed_hdu.header.comments['TTYPE3'] = 'Cross-coupling data'
feed_hdu.header.comments['TFORM3'] = 'NCPAR*NCHAN floats'
feed_hdu.header.comments['TTYPE4'] = 'Formal covariances of coupling data'
feed_hdu.header.comments['TFORM4'] = 'NCOVAR*NCHAN floats'
feed_hdu.header.comments[
'TTYPE5'] = 'Total chi-squared (objective merit function)'
feed_hdu.header.comments['TFORM5'] = 'NCHAN floats'
feed_hdu.header.comments['TTYPE6'] = 'Number of degrees of freedom'
feed_hdu.header.comments['TFORM6'] = 'NCHAN long (32-bit) integers'
feed_hdu.header['CAL_MTHD'] = ('van04e18', 'Cross-coupling method')
feed_hdu.header['NCPAR'] = ('7', 'Number of coupling parameters')
feed_hdu.header['NCOVAR'] = ('28', 'Number of parameter covariances')
feed_hdu.header['NCHAN'] = ('512', 'Nr of channels in Feed coupling data')
feed_hdu.header['EPOCH'] = ('56038.3352', '[MJD] Epoch of calibration obs')
feed_hdu.header['TUNIT1'] = ('MHz', 'Units of field')
feed_hdu.header['TDIM3'] = ('(7,512)', 'Dimensions (NCPAR,NCHAN)')
feed_hdu.header['TDIM4'] = ('(28,512)', 'Dimensions (NCOVAR,NCHAN)')
feed_hdu.header['EXTVER'] = ('1', 'auto assigned by template parser')
feed_hdu.header['PAR_0000'] = ('G', 'scalar gain')
feed_hdu.header['PAR_0001'] = ('gamma',
'differential gain (hyperbolic radians)')
feed_hdu.header['PAR_0002'] = ('phi', 'differential phase (radians)')
feed_hdu.header['PAR_0003'] = ('el0',
'ellipticity of receptor 0 (radians)')
feed_hdu.header['PAR_0004'] = ('or0',
'orientation of receptor 0 (radians)')
feed_hdu.header['PAR_0005'] = ('el1',
'ellipticity of receptor 1 (radians)')
feed_hdu.header['PAR_0006'] = ('or1',
'orientation of receptor 1 (radians)')
hdus = pyfits.HDUList(hdus=[prihdu, new_hdulist, feed_hdu])
hdus.writeto(outname, clobber=True)
def combineCats(catlist, saturation, instrum=None, mastercat=None):
#returns ldac cat
if len(catlist) == 0:
return
if len(catlist) == 1:
return cats[0]
referencecat = catlist[0]
fluxkeys, fluxerrkeys, otherkeys = _sortFluxKeys(referencecat.keys())
if mastercat is None:
mastercat = referencecat
else:
ignoreFluxkeys, ignoreFluxerrkeys, otherkeys = _sortFluxKeys(
mastercat.keys())
print fluxkeys
print fluxerrkeys
print otherkeys
cols = []
for key in otherkeys:
cols.append(_extractColumn(mastercat, key))
for fluxkey in fluxkeys:
fluxerr_key = _construct_fluxerr_key(fluxkey)
inputfluxs = []
inputerrs = []
inputflags = []
inputmaxvals = []
inputbackgr = []
saturations = []
for cat in catlist:
inputfluxs.append(cat[fluxkey])
inputerrs.append(cat[fluxerr_key])
inputflags.append(cat['IMAFLAGS_ISO'])
inputmaxvals.append(cat['MaxVal'])
inputbackgr.append(cat['BackGr'])
saturations.append(saturation)
fluxs = _combineFluxs(inputfluxs, inputerrs, inputflags, inputmaxvals,
inputbackgr, saturations)
for chipid, (flux, err) in fluxs.iteritems():
if instrum is None:
id = '%d' % chipid
else:
id = '%s-%d' % (instrum, chipid)
if len(flux.shape) == 1:
cols.append(
pyfits.Column(name='%s-%s' % (fluxkey, id),
format='E',
array=flux))
cols.append(
pyfits.Column(name='%s-%s' % (fluxerr_key, id),
format='E',
array=err))
else:
nelements = flux.shape[1]
cols.append(
pyfits.Column(name='%s-%s' % (fluxkey, id),
format='%dE' % nelements,
array=flux))
cols.append(
pyfits.Column(name='%s-%s' % (fluxerr_key, id),
format='%dE' % nelements,
array=err))
return ldac.LDACCat(
pyfits.new_table(pyfits.ColDefs(cols), header=mastercat.hdu.header))
# translate into energy bin lower and upper bounds and average response
n = len(ener)
emin = float32(ener[0:n-1])
emax = float32(ener[1:n])
resp = float32(area*0.5*(c1[0:n-1]+c1[1:n]))
plt.plot(0.5*(emin+emax), resp, '-k')
# make .arf file
# create FITS table with ARF data
col1 = pyfits.Column(name='E_MIN', format='1E', unit='keV', array=emin)
col2 = pyfits.Column(name='E_MAX', format='1E', unit='keV', array=emax)
col3 = pyfits.Column(name='SPECRESP', format='1E', unit='cm**2', array=resp)
# create a ColDefs (column-definitions) object for all columns:
cols = pyfits.ColDefs([col1, col2, col3])
# create a new binary table HDU object
arfhdu = pyfits.new_table(cols)
arfhdu.header.update('EXTNAME', 'SPECRESP', 'Extension type')
arfhdu.header.update('TELESCOP', 'HaloSat', 'Mission name')
arfhdu.header.update('INSTRUME', 'SDD1', 'Instrument')
#arfhdu.header.update('CHANTYPE', 'PHA', 'Channels from detector electronics')
#arfhdu.header.update('DETCHANS', len(emin), 'total number of raw detector PHA channels')
arfhdu.header.update('HDUCLASS', 'OGIP', 'file format is OGIP standard')
arfhdu.header.update('HDUCLAS1', 'RESPONSE', 'extension contains response data')
arfhdu.header.update('HDUCLAS2', 'SPECRESP', 'extension contains a response matrix')
arfhdu.header.update('HDUVERS', '1.1.0', 'version of the file format')
# create primary FITS header
prihdr = pyfits.Header()
prihdu = pyfits.PrimaryHDU(header=prihdr)
# create HDUList with primary HDU and the data table extension
subrealdat = pyfits.getdata(subrealfile)
realdat = pyfits.getdata(realfile)
fitdat = pyfits.getdata(fitfile)
if update_fits_cat:
# need to find the stuff to keep
match_ind = np.zeros(len(subrealdat)).astype(int)
for ind in range(len(subrealdat)):
match_ind[ind] = list(fitdat['ident']).index(subrealdat['ident'][ind])
new_dat = []
for ind in range(len(fitdat.columns)):
new_dat.append(pyfits.Column(name=fitdat.columns[ind].name,
format=fitdat.columns[ind].format,
array=fitdat[fitdat.columns[ind].name][match_ind]))
new_hdu = pyfits.new_table(pyfits.ColDefs(new_dat))
new_hdu.writeto(subfitfile,clobber=True)
if update_real_cat:
# need to find the stuff to keep
match_ind = np.zeros(len(subrealdat)).astype(int)
for ind in range(len(subrealdat)):
match_ind[ind] = list(realdat['ident']).index(subrealdat['ident'][ind])
new_dat = []
for ind in range(len(realdat.columns)):
# Only replace if we're not changing the filename / HDU.
if realdat.columns[ind].name not in \
['GAL_FILENAME', 'GAL_HDU', 'PSF_FILENAME', 'PSF_HDU', 'NOISE_FILENAME']:
new_dat.append(pyfits.Column(name=realdat.columns[ind].name,
format=realdat.columns[ind].format,
# store meta data
if sfh and met and imf:
primary_hdu.header.update('has_meta', True)
primary_hdu.header.update('model', 'BaSTI', comment='meta data')
primary_hdu.header.update('met', met, comment='meta data')
primary_hdu.header.update('imf', imf, comment='meta data')
primary_hdu.header.update('sfh', sfh, comment='meta data')
if sfh == 'Exponential':
primary_hdu.header.update('tau', tau, comment='meta data')
primary_hdu.header.update('n', n, comment='meta data')
primary_hdu.header.update('ae', ae, comment='meta data')
# store the list of frequencies in a table
vs_hdu = pyfits.new_table(
pyfits.ColDefs(
[pyfits.Column(name='vs', array=vs[sinds], format='D', unit='hertz')]))
vs_hdu.header.update('units', 'hertz')
# and the list of ages
cols = [pyfits.Column(name='ages', array=ages, format='D', unit='years')]
# and masses
if has_masses:
cols.append(
pyfits.Column(name='masses', array=masses, format='D', unit='m_sun'))
ages_hdu = pyfits.new_table(pyfits.ColDefs(cols))
if has_masses: ages_hdu.header.update('has_mass', True)
# make the fits file in memory
hdulist = pyfits.HDUList([primary_hdu, vs_hdu, ages_hdu])
# and write it out
hdulist.writeto(fileout, clobber=True)
def filter_catalogs(args,pointingras,pointingdecs,mosaiccat,outname,dessourcenums,cattype):
pointdirectories = args.pointdirectories
sourceids = np.array([])
sourceresolved = np.array([])
sourcera = np.array([])
e_sourcera = np.array([])
e_sourcera_tot = np.array([])
sourcedec = np.array([])
e_sourcedec = np.array([])
e_sourcedec_tot = np.array([])
sint = np.array([])
e_sint = np.array([])
e_sint_tot = np.array([])
speak = np.array([])
e_speak = np.array([])
e_speak_tot = np.array([])
maj = np.array([])
e_maj = np.array([])
smin = np.array([])
e_smin = np.array([])
pa = np.array([])
e_pa = np.array([])
rms_noise = np.array([])
stype = np.array([])
mosaic_identifier = np.array([])
gausid = np.array([])
islid = np.array([])
sourcenum = np.array([])
pointing = mosaiccat.replace('.cat.fits','-blanked.fits')
if cattype == 'gaus':
sourcecat = pyfits.open(mosaiccat)
mosaiccat = glob.glob(mosaiccat.replace('mosaic.cat.fits','mosaic-blanked_pybdsm/*/catalogues/mosaic-blanked.pybdsm.gaul.FITS'))[0]
cat = pyfits.open(mosaiccat)
f = pyfits.open(pointing)
rapointing = f[0].header['CRVAL1']*deg2rad
decpointing = f[0].header['CRVAL2']*deg2rad
f.close()
numsources = len(cat[1].data['RA'])
closepointingindex = np.where(sepn(pointingras,pointingdecs,rapointing,decpointing)*rad2deg < 5.0)
astromed = find_median_astrometry(pointdirectories,rapointing,decpointing)
keepindices = []
time1 = time.time()
for i in range(0,numsources):
if dessourcenums == None:
allsep = sepn(pointingras[closepointingindex],pointingdecs[closepointingindex],cat[1].data['RA'][i]*deg2rad,cat[1].data['DEC'][i]*deg2rad)
centsep = sepn(rapointing,decpointing,cat[1].data['RA'][i]*deg2rad,cat[1].data['DEC'][i]*deg2rad)
if min(allsep) != centsep:
continue
else:
keepindices.append(i)
else:
if cat[1].data['Source_id'][i] in dessourcenums:
keepindices.append(i)
else:
continue
if cattype == 'srl':
sc=SkyCoord(cat[1].data['RA'][i]*deg2rad*u.rad,cat[1].data['DEC'][i]*deg2rad*u.rad,frame='icrs')
if cattype == 'gaus':
sourceindex=cat[1].data['Source_id'][i]
sc=SkyCoord(sourcecat[1].data['RA'][sourceindex]*deg2rad*u.rad,sourcecat[1].data['DEC'][sourceindex]*deg2rad*u.rad,frame='icrs')
s=sc.to_string(style='hmsdms',sep='',precision=3)
identity = str('ILTJ'+s).replace(' ','')[:-1]
sourceids = np.append(sourceids,identity)
if cattype == 'srl':
mosaic_identifier = np.append(mosaic_identifier,mosaiccat.split('/')[-2])
if cattype == 'gaus':
mosaic_identifier = np.append(mosaic_identifier,mosaiccat.split('/')[-5])
fluxratio = cat[1].data['Total_flux'][i]/cat[1].data['Peak_flux'][i]
snr = cat[1].data['Peak_flux'][i]/cat[1].data['Isl_rms'][i]
# Some equation to figure out if the source is resolved -- leave these dummy values for now.
if fluxratio > (1.483 + 1000.4/(snr**3.94)):
#if fluxratio > (1.50341355 + 1.78467767/(snr**0.78385826)):
sourceresolved = np.append(sourceresolved,'R')
else:
sourceresolved = np.append(sourceresolved,'U')
print 'Keeping %s sources for %s -- took %s'%(len(keepindices),pointing,time.time()-time1)
sourcera = np.append(sourcera,cat[1].data[keepindices]['RA'])
e_sourcera = np.append(e_sourcera,cat[1].data[keepindices]['E_RA'])
e_sourcera_tot = np.append(e_sourcera_tot,(cat[1].data[keepindices]['E_RA']**2.0 + (astromed*arcsec2deg)**2.0)**0.5) #$ ADD SOME ERROR TO THE SOURCE POSITIONS
sourcedec = np.append(sourcedec,cat[1].data[keepindices]['DEC'])
e_sourcedec = np.append(e_sourcedec,cat[1].data[keepindices]['E_DEC'])
e_sourcedec_tot = np.append(e_sourcedec_tot,(cat[1].data[keepindices]['E_DEC']**2.0 + (astromed*arcsec2deg)**2.0)**0.5) # ADD SOME ERROR TO THE SOURCE POSITIONS
sint = np.append(sint,cat[1].data[keepindices]['Total_flux'])
e_sint =np.append(e_sint,cat[1].data[keepindices]['E_Total_flux'])
e_sint_tot =np.append(e_sint_tot,(cat[1].data[keepindices]['E_Total_flux']**2.0 + (cat[1].data[keepindices]['Total_flux']*0.2)**2.0)**0.5)
speak = np.append(speak,cat[1].data[keepindices]['Peak_flux'])
e_speak =np.append(e_speak,cat[1].data[keepindices]['E_Peak_flux'])
e_speak_tot =np.append(e_speak_tot,(cat[1].data[keepindices]['E_Peak_flux']**2.0 + (cat[1].data[keepindices]['Peak_flux']*0.2)**2.0)**0.5)
maj = np.append(maj,cat[1].data[keepindices]['Maj'])
e_maj =np.append(e_maj,cat[1].data[keepindices]['E_Maj'])
smin = np.append(smin,cat[1].data[keepindices]['Min'])
e_smin = np.append(e_smin,cat[1].data[keepindices]['E_Min'])
pa = np.append(pa,cat[1].data[keepindices]['PA'])
e_pa = np.append(e_pa,cat[1].data[keepindices]['E_PA'])
rms_noise = np.append(rms_noise,cat[1].data[keepindices]['Isl_rms'])
stype = np.append(stype,cat[1].data[keepindices]['S_Code'])
islid = np.append(islid,cat[1].data[keepindices]['Isl_id'])
sourcenum = np.append(sourcenum,cat[1].data[keepindices]['Source_id'])
col1 = pyfits.Column(name='Source_Name',format='24A',unit='',array=sourceids)
col2 = pyfits.Column(name='RA',format='f8',unit='deg',array=sourcera)
col3 = pyfits.Column(name='E_RA',format='f8',unit='arcsec',array=e_sourcera*deg2arcsec)
col4 = pyfits.Column(name='E_RA_tot',format='f8',unit='arcsec',array=e_sourcera_tot*deg2arcsec)
col5 = pyfits.Column(name='DEC',format='f8',unit='deg',array=sourcedec)
col6 = pyfits.Column(name='E_DEC',format='f8',unit='arcsec',array=e_sourcedec*deg2arcsec)
col7 = pyfits.Column(name='E_DEC_tot',format='f8',unit='arcsec',array=e_sourcedec_tot*deg2arcsec)
col8 = pyfits.Column(name='Peak_flux',format='f8',unit='beam-1 mJy',array=speak*1000.0)
col9 = pyfits.Column(name='E_Peak_flux',format='f8',unit='beam-1 mJy',array=e_speak*1000.0)
col10 = pyfits.Column(name='E_Peak_flux_tot',format='f8',unit='beam-1 mJy',array=e_speak_tot*1000.0)
col11 = pyfits.Column(name='Total_flux',format='f8',unit='mJy',array=sint*1000.0)
col12 = pyfits.Column(name='E_Total_flux',format='f8',unit='mJy',array=e_sint*1000.0)
col13 = pyfits.Column(name='E_Total_flux_tot',format='f8',unit='mJy',array=e_sint_tot*1000.0)
#maj[np.where(sourceresolved=='U')] = np.nan
#e_maj[np.where(sourceresolved=='U')] = np.nan
#smin[np.where(sourceresolved=='U')] = np.nan
#e_smin[np.where(sourceresolved=='U')] = np.nan
#pa[np.where(sourceresolved=='U')] = np.nan
#e_pa[np.where(sourceresolved=='U')] = np.nan
col14 = pyfits.Column(name='Maj',format='f8',unit='arcsec',array=maj*deg2arcsec)
col15 = pyfits.Column(name='E_Maj',format='f8',unit='arcsec',array=e_maj*deg2arcsec)
col16 = pyfits.Column(name='Min',format='f8',unit='arcsec',array=smin*deg2arcsec)
col17 = pyfits.Column(name='E_Min',format='f8',unit='arcsec',array=e_smin*deg2arcsec)
col18 = pyfits.Column(name='PA',format='f8',unit='deg',array=pa)
col19 = pyfits.Column(name='E_PA',format='f8',unit='deg',array=e_pa)
#col20 = pyfits.Column(name='Resolved',format='1A',unit='',array=sourceresolved)
col20 = pyfits.Column(name='Isl_rms',format='f8',unit='beam-1 mJy',array=rms_noise*1000.0)
col21 = pyfits.Column(name='S_Code',format='1A',unit='',array=stype)
col22 = pyfits.Column(name='Mosaic_ID',format='8A',unit='',array=mosaic_identifier)
col23 = pyfits.Column(name='Isl_id',format='I8',unit='',array=islid)
# With unique source names that are matched with source and gaussian catalogs the source_id is not needed.
#col24 = pyfits.Column(name='Source_id',format='I8',unit='',array=sourcenum)
if cattype == 'gaus':
gausid = np.append(gausid,cat[1].data[keepindices]['Gaus_id'])
col24 = pyfits.Column(name='Gaus_id',format='I8',unit='',array=gausid)
if cattype == 'srl':
cols = pyfits.ColDefs([col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23])
if cattype == 'gaus':
cols = pyfits.ColDefs([col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24])
tbhdu = pyfits.BinTableHDU.from_columns(cols)
if cattype == 'gaus':
regionfile = open('%s.gaus.reg'%outname,'w')
if cattype == 'srl':
regionfile = open('%s.srl.reg'%outname,'w')
regionfile.write('# Region file format: DS9 version 4.0\n')
regionfile.write('global color=green font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n')
regionfile.write('fk5\n')
for i in range(0,len(sourceids)):
if not np.isnan(maj[i]):
regionfile.write('ellipse(%s,%s,%s,%s,%s)\n'%(sourcera[i],sourcedec[i],maj[i],smin[i],pa[i]+90))
else:
regionfile.write('box(%s,%s,5.0",5.0",0.0)\n'%(sourcera[i],sourcedec[i]))
regionfile.close()
prihdr = pyfits.Header()
prihdr['NAME'] = outname
prihdu = pyfits.PrimaryHDU(header=prihdr)
tbhdulist = pyfits.HDUList([prihdu, tbhdu])
if cattype == 'srl':
outcat = outname +'.srl.fits'
if cattype == 'gaus':
outcat = outname +'.gaus.fits'
tbhdulist.writeto(outcat)
return sourcenum,outcat
def filt_curve(agn):
#for i in range(1,30):
print agn
arch=pf.open(lc_path+agn)
datos=arch[1].data
head0=arch[0].header
mag=datos['Q']
errmag=datos['errQ']
jd0=datos['JD']
catalog=datos['catalog']
chip=datos['chip']
tile=datos['tile']
fit_rms=datos['fit_rms']
chi_red=datos['chi_red']
num_stars=datos['num_stars']
m=np.where((mag<30) & (chi_red<=20) & (fit_rms<=0.1) & (num_stars>=20))
jd=jd0[m]
mag1=mag[m]
err1=errmag[m]
catalog1=catalog[m]
chip1=chip[m]
tile1=tile[m]
fit_rms1=fit_rms[m]
chi_red1=chi_red[m]
num_stars1=num_stars[m]
errmedian1=np.median(err1)
l=np.where((err1<2.0*errmedian1))
jd=jd[l]
mag1=mag1[l]
err1=err1[l]
catalog1=catalog1[l]
chip1=chip1[l]
tile1=tile1[l]
fit_rms1=fit_rms1[l]
chi_red1=chi_red1[l]
num_stars1=num_stars1[l]
#se ajusta un polinomio de orden 5.
if len(jd)>0:
coefficients = np.polyfit(jd, mag1, 5)
polynomial = np.poly1d(coefficients)
pol=polynomial(jd)
#se calcula el rms
rm=(mag1-pol)**2
rms=np.sum(rm)
rms=rms/(len(jd))
#se hace el filtro por 3 sigmas
dist=np.abs(mag1-pol)
sigma=np.sqrt(rms+err1**2)
n=np.where((dist/sigma)<=1.5)
dia=jd[n]
magf1=mag1[n]
errf1=err1[n]
catalog1=catalog1[n]
chip1=chip1[n]
tile1=tile1[n]
fit_rms1=fit_rms1[n]
chi_red1=chi_red1[n]
num_stars1=num_stars1[n]
dif=len(jd0)-len(dia)
print dif
fluxf2=10**(-0.4*(magf1+48.6))
errflux2=0.4*np.log(10)*(10.0**(-0.4*(magf1+48.6)))*errf1
list_chips=list(set(chip1))
num_chips=len(list_chips)
num_tiles=len(set(tile1))
#save the data for sources with detections in only one chip
if (num_chips==1):
#se guardan los fits
c1=pf.Column(name='JD',format='D',array=dia)
c2=pf.Column(name='Q',format='D',array=magf1)
c3=pf.Column(name='errQ',format='D',array=errf1)
c4=pf.Column(name='fluxQ',format='D',array=fluxf2)
c5=pf.Column(name='errfluxQ',format='D',array=errflux2)
c6=pf.Column(name='catalog',format='32A',array=catalog1)
c7=pf.Column(name='chip',format='3A',array=chip1)
c8=pf.Column(name='tile',format='E',array=tile1)
c9=pf.Column(name='fit_rms',format='E',array=fit_rms1)
c10=pf.Column(name='chi_red',format='E',array=chi_red1)
c11=pf.Column(name='num_stars',format='E',array=num_stars1)
coldef=pf.ColDefs([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11])
thdu=pf.new_table(coldef)
thdu.writeto(bin_name+'_onechip_'+agn)
arch1=pf.open(bin_name+'_onechip_'+agn,mode='update')
head=arch1[0].header
head['ALPHA']=head0['ALPHA']
head['DELTA']=head0['DELTA']
head['uMAG']=head0['uMAG']
head['uERR']= head0['uERR']
head['gMAG']= head0['gMAG']
head['gERR']= head0['gERR']
head['rMAG']= head0['rMAG']
head['rERR']= head0['rERR']
head['iMAG']= head0['iMAG']
head['iERR']= head0['iERR']
head['zMAG']= head0['zMAG']
head['zERR']= head0['zERR']
head['DELPOINT']= dif
head['FILTER']= 'Q'
head['T_RANGE']= dia[-1]-dia[0]
head['T_length']= len(dia)
head['CHIP']= chip1[0]
head['TILE']= tile1[0]
arch1.flush()
cmd='mv '+bin_name+'_onechip_'+agn+' '+lc_path
print "num epochs= %d" % (len(dia))
print "lc length= %f" % (dia[-1]-dia[0])
print "num chips= %d" % num_chips
os.system(cmd)
#save the data for sources with detections in only one chip
elif (num_chips>1):
nchip=np.arange(len(chip1))
if num_chips==2:
lchip1=len(np.where(chip1==list_chips[0])[0])
lchip2=len(np.where(chip1==list_chips[1])[0])
lchiptot=len(chip1)
fracchip1=float(lchip1)/float(lchiptot)
fracchip2=float(lchip2)/float(lchiptot)
if (fracchip1>=0.2) and (fracchip2>=0.2):
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2):
nchip=np.where((chip1==list_chips[0]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2):
nchip=np.where((chip1==list_chips[1]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif num_chips==3:
lchip1=len(np.where(chip1==list_chips[0])[0])
lchip2=len(np.where(chip1==list_chips[1])[0])
lchip3=len(np.where(chip1==list_chips[2])[0])
lchiptot=len(chip1)
fracchip1=float(lchip1)/float(lchiptot)
fracchip2=float(lchip2)/float(lchiptot)
fracchip3=float(lchip3)/float(lchiptot)
if (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3<0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3>=0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) :
nchip=np.where((chip1==list_chips[1]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3<0.2) :
nchip=np.where((chip1==list_chips[0]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3<0.2) :
nchip=np.where((chip1==list_chips[1]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2<0.2) and (fracchip3>=0.2) :
nchip=np.where((chip1==list_chips[2]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif num_chips==4:
lchip1=len(np.where(chip1==list_chips[0])[0])
lchip2=len(np.where(chip1==list_chips[1])[0])
lchip3=len(np.where(chip1==list_chips[2])[0])
lchip4=len(np.where(chip1==list_chips[3])[0])
lchiptot=len(chip1)
fracchip1=float(lchip1)/float(lchiptot)
fracchip2=float(lchip2)/float(lchiptot)
fracchip3=float(lchip3)/float(lchiptot)
fracchip4=float(lchip4)/float(lchiptot)
if (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) and (fracchip4>=0.2):
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]) | (chip1==list_chips[2]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3<0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3>=0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[2]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[1]) | (chip1==list_chips[2]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2>=0.2) and (fracchip3<0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[1]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3>=0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3<0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[0]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3>=0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[1]) | (chip1==list_chips[2]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3<0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[1]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1<0.2) and (fracchip2<0.2) and (fracchip3>=0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[2]) | (chip1==list_chips[3]))
errnew=0.05
new_name=bin_name+'_morechip_'+agn
elif (fracchip1>=0.2) and (fracchip2<0.2) and (fracchip3<0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[0]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2>=0.2) and (fracchip3<0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[1]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2<0.2) and (fracchip3>=0.2) and (fracchip4<0.2) :
nchip=np.where((chip1==list_chips[2]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
elif (fracchip1<0.2) and (fracchip2<0.2) and (fracchip3<0.2) and (fracchip4>=0.2) :
nchip=np.where((chip1==list_chips[3]))
errnew=0.0
new_name=bin_name+'_onechip_'+agn
else:
nchip=np.arange(0,len(chip1),1)
errnew=0.05
new_name=bin_name+'_severalchip_'+agn
dia=dia[nchip]
magf1=magf1[nchip]
errf1=errf1[nchip]
catalog1=catalog1[nchip]
chip1=chip1[nchip]
tile1=tile1[nchip]
fit_rms1=fit_rms1[nchip]
chi_red1=chi_red1[nchip]
num_stars1=num_stars1[nchip]
fluxf2=10**(-0.4*(magf1+48.6))
errf1=np.sqrt(errf1**2+(errnew)**2)
errflux2=0.4*np.log(10)*(10.0**(-0.4*(magf1+48.6)))*errf1
#se guardan los fits
c1=pf.Column(name='JD',format='D',array=dia)
c2=pf.Column(name='Q',format='D',array=magf1)
c3=pf.Column(name='errQ',format='D',array=errf1)
c4=pf.Column(name='fluxQ',format='D',array=fluxf2)
c5=pf.Column(name='errfluxQ',format='D',array=errflux2)
c6=pf.Column(name='catalog',format='32A',array=catalog1)
c7=pf.Column(name='chip',format='3A',array=chip1)
c8=pf.Column(name='tile',format='D',array=tile1)
c9=pf.Column(name='fit_rms',format='D',array=fit_rms1)
c10=pf.Column(name='chi_red',format='D',array=chi_red1)
c11=pf.Column(name='num_stars',format='D',array=num_stars1)
coldef=pf.ColDefs([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11])
thdu=pf.new_table(coldef)
thdu.writeto(new_name)
arch1=pf.open(new_name,mode='update')
head=arch1[0].header
head['ALPHA']=head0['ALPHA']
head['DELTA']=head0['DELTA']
head['uMAG']=head0['uMAG']
head['uERR']= head0['uERR']
head['gMAG']= head0['gMAG']
head['gERR']= head0['gERR']
head['rMAG']= head0['rMAG']
head['rERR']= head0['rERR']
head['iMAG']= head0['iMAG']
head['iERR']= head0['iERR']
head['zMAG']= head0['zMAG']
head['zERR']= head0['zERR']
head['DELPOINT']= dif
head['FILTER']= 'Q'
head['T_RANGE']= dia[-1]-dia[0]
head['T_length']= len(dia)
arch1.flush()
cmd='mv '+new_name+' '+lc_path
print "num epochs= %d" % (len(dia))
print "lc length= %f" % (dia[-1]-dia[0])
print "num chips= %d" % num_chips
if len(dia)>10: os.system(cmd)
else: os.system('rm '+new_name)
arch.close()
## -- Read the input spec -- ##
f1 = pyfits.open(grp1)
try:
g1 = f1['SPECTRUM'].data.field('GROUPING')
except:
print 'I could not read the grouping from {}'.format(grp1)
exit(0)
f1.close()
## -------------------------- ##
## -- write the binning to the output file -- ##
#from IPython import embed
#embed();exit(0)
f2 = pyfits.open(grp2)
hdu = f2['SPECTRUM']
orig_cols = hdu.columns
new_cols = pyfits.ColDefs(
[pyfits.Column(name='GROUPING', format='1I', array=g1)])
try:
tbl = pyfits.BinTableHDU.from_columns(orig_cols + new_cols)
except:
orig_cols.del_col('GROUPING')
tbl = pyfits.BinTableHDU.from_columns(orig_cols + new_cols)
hdu.header.update(tbl.header.copy())
tbl.header = hdu.header.copy()
grp = pyfits.HDUList([f2[0], tbl])
os.system('rm {0} &> /dev/null'.format(out))
grp.writeto(out)
f2.close()
## ------------------------------------------ ##
def extract_spectrum(sourceIDs,
layer_scale_arr,
wavelengths,
noise_cube=None,
source_model_cube=None,
specname='tdose_extract_spectra_extractedspec.fits',
obj_cube_hdr=None,
data_cube=None,
clobber=False,
verbose=True):
"""
Extracting a spectrum based on the layer scale image from the model cube provided a list of sources to combine.
Noise is estimated from the noise cube (of the data)
If all layer_scales are 1 a data_cube for the extractions is expected
--- INPUT ---
sourceIDs The source IDs to combine into spectrum
layer_scale_arr Layer scale array (or image) produced when generating the model cube
fractional flux belonging to the source in each pixel
wavelengths Wavelength vector to use for extracted 1D spectrum.
noise_cube Cube with uncertainties (sqrt(variance)) of data cube to be used for estimating 1D uncertainties
To estimate S/N and 1D noise, providing a source model cube is required
source_model_cube Source model cube containing the model cube for each individual source seperately
Needed in order to estimate noise from noise-cube
specname Name of file to save spectrum to
obj_cube_hdr Provide a template header to save the object cube (from combining the individual source cube)
as an extension to the extracted spectrum
data_cube In case all layers scales are 1, it is assumed that the source_model_cube contains a mask for the
spectral extraction, which will then be performed on this data_cube.
clobber To overwrite existing files set clobber=True
verbose Toggle verbosity
--- EXAMPLE OF USE ---
"""
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose: print ' - Checking shape of wavelengths and layer_scale_arr'
if wavelengths.shape[0] != layer_scale_arr.shape[1]:
sys.exit(' ---> Shape of wavelength vector (' +
str(wavelengths.shape) +
') and wavelength dimension of layer scale array (' +
layer_scale_arr.shape[1].shape + ') do not match.')
else:
if verbose: print ' dimensions match; proceeding...'
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose: print ' - Checking all sources have spectra in layer_scale_arr'
maxsource = np.max(sourceIDs)
if maxsource >= layer_scale_arr.shape[0]:
sys.exit(' ---> Sources in list ' + str(str(sourceIDs)) +
' not available among ' + str(layer_scale_arr.shape[0]) +
' sources in layer_scale_arr.')
else:
if verbose:
print ' All sources exist in layer_scale_arr; proceeding...'
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose: print ' - Assembling object spectrum from source scaling'
source_ent = np.asarray(sourceIDs).astype(int)
if (layer_scale_arr == 1).all():
if verbose:
print ' - All layer scales are 1; assuming source model cube contain mask for spectral extraction'
object_cube = np.sum(np.abs(source_model_cube[source_ent, :, :]),
axis=0)
if data_cube is None:
sys.exit(
' ---> Did not find a data cube to extrac spectra from as expected'
)
object_mask = (object_cube == 0) # masking all zeros in object mask
invalid_mask = np.ma.masked_invalid(data_cube).mask
comb_mask = (invalid_mask | object_mask)
spec_1D_masked = np.sum(np.sum(np.ma.array(data_cube, mask=comb_mask),
axis=1),
axis=1)
spec_1D = spec_1D_masked.filled(fill_value=0.0)
if noise_cube is not None:
if verbose:
print ' Calculating noise as d_spec_k = sqrt( SUMij d_pix_ij**2 ), i.e., as the sqrt of variances summed'
invalid_mask_noise = np.ma.masked_invalid(noise_cube).mask
comb_mask = (comb_mask | invalid_mask_noise)
variance_1D_masked = np.ma.array(noise_cube, mask=comb_mask)**2
noise_1D_masked = np.sqrt(
np.sum(np.sum(variance_1D_masked, axis=1), axis=1))
noise_1D = noise_1D_masked.filled(fill_value=np.nan)
if verbose: print ' Generating S/N vector'
SN_1D = spec_1D / noise_1D
else:
if verbose:
print ' - No "noise_cube" provided. Setting all errors and S/N values to NaN'
SN_1D = np.zeros(spec_1D.shape) * np.NaN
noise_1D = np.zeros(spec_1D.shape) * np.NaN
else:
if verbose:
print ' - Some layer scales are different from 1; hence assembling spectra using layer scales'
if len(source_ent) < 1:
spec_1D = layer_scale_arr[source_ent, :]
else:
spec_1D = np.sum(layer_scale_arr[source_ent, :], axis=0)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if noise_cube is not None:
if verbose:
print ' - Estimate S/N at each wavelength for 1D spectrum (see Eq. 16 of Kamann+2013)'
if verbose:
print ' Estimating fraction of flux in each pixel wrt. total flux in each layer'
object_cube = np.sum(
(source_model_cube[source_ent, :, :, :]),
axis=0) # summing source models for all source IDs
fluxfrac_cube_sents = np.zeros(source_model_cube.shape[1:])
for sent in source_ent:
object_cube_sent = np.sum(
(source_model_cube[[sent], :, :, :]),
axis=0) # getting source model for model 'sent'
fluxscale1D_sent = layer_scale_arr[sent, :]
fluxfrac_cube_sent = object_cube_sent / fluxscale1D_sent[:,
None,
None]
fluxfrac_cube_sents = fluxfrac_cube_sents + fluxfrac_cube_sent
fluxfrac_cube = fluxfrac_cube_sents / len(
source_ent) # renormalizing flux-fraction cube
if verbose:
print ' Defining pixel mask (ignoring NaN pixels) ' #+\
# 'and pixels with <'+str(fluxfrac_min)+' of total pixel flux in model cube) '
# pix_mask = (fluxfrac_cube < fluxfrac_min)
invalid_mask1 = np.ma.masked_invalid(fluxfrac_cube).mask
invalid_mask2 = np.ma.masked_invalid(noise_cube).mask
# combining mask making sure all individual mask pixels have True for it to be true in combined mask
comb_mask = (invalid_mask1 | invalid_mask2) # | pix_mask
if verbose:
print ' Calculating noise propogated as d_spec_k = 1/sqrt( SUMij (fluxfrac_ij**2 / d_pix_ij**2) )'
squared_ratio = np.ma.array(fluxfrac_cube,
mask=comb_mask)**2 / np.ma.array(
noise_cube, mask=comb_mask)**2
inv_noise_masked = np.sqrt(
np.sum(np.sum(squared_ratio, axis=1), axis=1))
noise_1D = (1.0 / inv_noise_masked).filled(fill_value=0.0)
if verbose: print ' Generating S/N vector'
SN_1D = spec_1D / noise_1D
else:
if verbose:
print ' - No "noise_cube" provided. Setting all errors and S/N values to NaN'
SN_1D = np.zeros(spec_1D.shape) * np.NaN
noise_1D = np.zeros(spec_1D.shape) * np.NaN
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if verbose:
print ' - Saving extracted 1D spectrum and source cube to \n ' + specname
mainHDU = pyfits.PrimaryHDU() # primary HDU
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
c1 = pyfits.Column(name='wave',
format='D',
unit='ANGSTROMS',
array=wavelengths)
c2 = pyfits.Column(name='flux', format='D', unit='', array=spec_1D)
c3 = pyfits.Column(name='fluxerror', format='D', unit='', array=noise_1D)
c4 = pyfits.Column(name='s2n', format='D', unit='', array=SN_1D)
coldefs = pyfits.ColDefs([c1, c2, c3, c4])
th = pyfits.new_table(coldefs) # creating default header
# writing hdrkeys:'---KEY--', '----------------MAX LENGTH COMMENT-------------'
th.header.append(('EXTNAME ', 'SPEC1D', 'cube containing source'),
end=True)
head = th.header
tbHDU = pyfits.new_table(coldefs, header=head)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if obj_cube_hdr is not None:
objHDU = pyfits.ImageHDU(object_cube)
for hdrkey in obj_cube_hdr.keys():
if not hdrkey in objHDU.header.keys():
objHDU.header.append((hdrkey, obj_cube_hdr[hdrkey],
obj_cube_hdr.comments[hdrkey]),
end=True)
try:
objHDU.header.append(('EXTNAMEC', objHDU.header['EXTNAME'],
'EXTNAME of original source cube'),
end=True)
del objHDU.header['EXTNAME']
except:
pass
objHDU.header.append(
('EXTNAME ', 'SOURCECUBE', 'cube containing source'), end=True)
hdulist = pyfits.HDUList([mainHDU, tbHDU, objHDU])
else:
hdulist = pyfits.HDUList([mainHDU, tbHDU])
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
hdulist.writeto(specname, clobber=clobber)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
return wavelengths, spec_1D, noise_1D, object_cube
final_pol = polynomial
final_fit = final_pol(magQi)
Q_cal = magQi - final_fit
Q_err = np.sqrt(errmagQi**2 + rms)
#writing the new catalogs:
c1 = pf.Column(name='ALPHA_J2000', format='D', array=alpha)
c2 = pf.Column(name='DELTA_J2000', format='D', array=delta)
c3 = pf.Column(name='MAG_2', format='D', array=Q_cal)
c4 = pf.Column(name='MAGERR_2', format='D', array=Q_err)
c5 = pf.Column(name='FLAGS', format='D', array=flagsf)
c6 = pf.Column(name='CLASS_STAR', format='D', array=clas)
c7 = pf.Column(name='FWHM_IMAGE', format='D', array=fwhm)
c8 = pf.Column(name='MAG_inst', format='D', array=magQi)
coldef = pf.ColDefs([c1, c2, c3, c4, c5, c6, c7, c8])
thdu = pf.new_table(coldef)
thdu.writeto(fold_catalogs + namelist[i] + '.calib.fits')
arch1 = pf.open(fold_catalogs + namelist[i] + '.calib.fits',
mode='update')
head = arch1[0].header
head['Field'] = field
head['fit_rms'] = np.sqrt(rms)
head['chi_red'] = chi_red
head['num_stars'] = len(maginst)
arch1.flush()
del arch1
print 'catalog', namelist[i], 'generated ###########'
def write_recarray(recarr,
filename,
header_dict={},
units={},
ext='new',
close=True):
"""
Write or add a record array to a FITS file.
If 'filename' refers to an existing file, the record array will be added
(ext='new') to the HDUlist or replace an existing HDU (ext=integer). Else,
a new file will be created.
Units can be given as a dictionary with keys the same names as the column
names of the record array.
A header_dictionary can be given, it is used to update an existing header
or create a new one if the extension is new.
"""
is_file = isinstance(filename, str) and os.path.isfile(filename)
if isinstance(filename, str) and not os.path.isfile(filename):
primary = np.array([[0]])
hdulist = pyfits.HDUList([pyfits.PrimaryHDU(primary)])
hdulist.writeto(filename)
hdulist.close()
if is_file or isinstance(filename, str):
hdulist = pyfits.open(filename, mode='update')
else:
hdulist = filename
#-- create the table HDU
cols = []
for i, name in enumerate(recarr.dtype.names):
format = recarr.dtype[i].str.lower().replace('|', '').replace(
's', 'a').replace('>', '')
format = format.replace('b1', 'L').replace('<', '')
unit = name in units and units[name] or 'NA'
cols.append(
pyfits.Column(name=name,
format=format,
array=recarr[name],
unit=unit))
tbhdu = pyfits.new_table(pyfits.ColDefs(cols))
#-- take care of the header:
if len(header_dict):
for key in header_dict:
if (len(key) > 8) and (not key in tbhdu.header.keys()) and (
not key[:9] == 'HIERARCH'):
key_ = 'HIERARCH ' + key
else:
key_ = key
tbhdu.header.update(key_, header_dict[key])
if ext != 'new':
tbhdu.header.update('EXTNAME', ext)
# put it in the right place
extnames = [
iext.header['EXTNAME'] for iext in hdulist
if ('extname' in iext.header.keys()) or (
'EXTNAME' in iext.header.keys())
]
if ext == 'new' or not ext in extnames:
logger.info('Creating new extension %s' % (ext))
hdulist.append(tbhdu)
ext = -1
else:
logger.info('Overwriting existing extension %s' % (ext))
hdulist[ext] = tbhdu
if close:
hdulist.close()
return filename
else:
return hdulist
def makeStarCats(self):
### star catalogs
seqnr = np.arange(501, 601)
pickles = ldac.openObjectFile('Pickles.cat', 'PICKLES')
pickles_sdss = ldac.openObjectFile('Pickles.cat', 'SDSS')
sample = np.random.randint(0, len(pickles), len(seqnr))
filter_cols = {}
for filter, zp in self.targetZPs.iteritems():
if filter == 'SUBARU-COADD-1-W-J-V':
raw_mag = pickles['SUBARU-10_2-1-W-J-V'][sample]
else:
raw_mag = pickles[filter][sample]
raw_flux = 10**(-0.4 * (raw_mag - zp))
fluxerr = 0.002 * raw_flux
flux = raw_flux + fluxerr * np.random.standard_normal(len(sample))
mags, magerrs = measure_unstacked_photometry.calcMags(
flux, fluxerr)
filter_cols[filter] = [mags, magerrs, flux, fluxerr]
### star cat
cols = []
cols.append(pyfits.Column(name='SeqNr', format='J', array=seqnr))
cols.append(
pyfits.Column(name='ALPHA_J2000',
format='E',
array=152.136393 * np.ones_like(seqnr)))
cols.append(
pyfits.Column(name='DELTA_J2000',
format='E',
array=56.703567 * np.ones_like(seqnr)))
for filter, (mags, magerrs, fluxs,
fluxerrs) in filter_cols.iteritems():
cols.append(
pyfits.Column(name='MAG_%s-%s' % (self.fluxtype, filter),
format='E',
array=mags))
cols.append(
pyfits.Column(name='MAGERR_%s-%s' % (self.fluxtype, filter),
format='E',
array=magerrs))
cols.append(
pyfits.Column(name='FLUX_%s-%s' % (self.fluxtype, filter),
format='E',
array=fluxs))
cols.append(
pyfits.Column(name='FLUXERR_%s-%s' % (self.fluxtype, filter),
format='E',
array=fluxerrs))
hdu = pyfits.new_table(pyfits.ColDefs(cols))
hdu.header.update('EXTNAME', 'OBJECTS')
hdu.writeto(self.star_cat_file, clobber=True)
### matched catalog
cols = []
cols.append(pyfits.Column(name='SeqNr', format='J', array=seqnr))
for filter, (mags, magerrs, fluxs,
fluxerrs) in filter_cols.iteritems():
cols.append(
pyfits.Column(name='SEx_MAG_%s-%s' % (self.fluxtype, filter),
format='E',
array=mags))
cols.append(
pyfits.Column(name='SEx_MAGERR_%s-%s' %
(self.fluxtype, filter),
format='E',
array=magerrs))
cols.append(
pyfits.Column(name='SEx_FLUX_%s-%s' % (self.fluxtype, filter),
format='E',
array=fluxs))
cols.append(
pyfits.Column(name='SEx_FLUXERR_%s-%s' %
(self.fluxtype, filter),
format='E',
array=fluxerrs))
cols.append(
pyfits.Column(name='Clean', format='J',
array=np.zeros_like(seqnr)))
cols.append(
pyfits.Column(name='SEx_BackGr',
format='E',
array=np.ones_like(seqnr)))
cols.append(
pyfits.Column(name='SEx_MaxVal',
format='E',
array=np.ones_like(seqnr)))
cols.append(
pyfits.Column(name='SEx_Flag',
format='J',
array=np.zeros_like(seqnr)))
cols.append(
pyfits.Column(name='SEx_Xpos',
format='E',
array=np.random.uniform(0, 5000, len(seqnr))))
cols.append(
pyfits.Column(name='SEx_Ypos',
format='E',
array=np.random.uniform(0, 5000, len(seqnr))))
for sdss in 'u g r i z'.split():
cols.append(
pyfits.Column(name='%smag' % sdss,
format='E',
array=pickles_sdss['%sp' % sdss][sample]))
cols.append(
pyfits.Column(name='%serr' % sdss,
format='E',
array=0.2 * pickles_sdss['%sp' % sdss][sample]))
for color in 'umg gmr rmi imz'.split():
f1, f2 = color.split('m')
cols.append(
pyfits.Column(name=color,
format='E',
array=pickles_sdss['%sp' % f1][sample] -
pickles_sdss['%sp' % f2][sample]))
cols.append(
pyfits.Column(name='%serr' % color,
format='E',
array=0.2 * np.ones_like(seqnr)))
hdu = pyfits.new_table(pyfits.ColDefs(cols))
hdu.header.update('EXTNAME', 'PSSC')
hdu.writeto(self.matched_catalog_file, clobber=True)
' ' + str(z_col[i]) + '\n')
zcat.close()
cols = []
cols.append(
pyfits.Column(name='Nr', format='J', array=scipy.array(SeqNr_col)))
cols.append(pyfits.Column(name='Ra', format='D',
array=scipy.array(ra_col)))
cols.append(
pyfits.Column(name='Dec', format='D', array=scipy.array(dec_col)))
cols.append(pyfits.Column(name='Z', format='D', array=scipy.array(z_col)))
cols.append(
pyfits.Column(name='FIELD_POS',
format='J',
array=scipy.ones(len(z_col))))
coldefs = pyfits.ColDefs(cols)
OBJECTS = pyfits.new_table(coldefs)
OBJECTS.header.update('extname', 'OBJECTS')
cols = []
cols.append(pyfits.Column(name='OBJECT_POS', format='J', array=[1.]))
cols.append(
pyfits.Column(name='OBJECT_COUNT', format='D', array=[len(z_col)]))
cols.append(pyfits.Column(name='CHANNEL_NR', format='J', array=[0]))
coldefs = pyfits.ColDefs(cols)
FIELDS = pyfits.new_table(coldefs)
FIELDS.header.update('extname', 'FIELDS')
print 'writing out fits file '
import os
thdulist = pyfits.HDUList([hdu, OBJECTS, FIELDS])
def simulate_pulse_random(self, exposure_ks):
sys.stdout.write('[simulate_pulse_random] \n')
# http://pacocat.com/?p=596
exposure = exposure_ks * 1e+3
self.config['number_of_pulsed_photons'] = int(exposure * self.config['source_rate'] * self.config['pulsed_fraction'])
self.config['number_of_DC_photons'] = int(exposure * (self.config['background_rate'] + self.config['source_rate'] * (1.-self.config['pulsed_fraction'])))
self.config['number_of_background_photons'] = int(exposure * self.config['background_rate'])
self.config['number_of_photons'] = self.config['number_of_pulsed_photons'] + self.config['number_of_DC_photons']
func = interp1d(self.out_phase,self.out_pulse_profle,kind='linear') # interpolate
func_range = [min(self.out_phase),max(self.out_phase)] # holizontal range [x_min, x_max]
func_max = max(self.out_pulse_profle) # vertical range
tstart_list = []
tstop_list = []
gti_list = []
n = 0; exp = 0.; on_source = 0.0
while exp <= exposure:
tstart = n * self.config['orbit_period_min'] * 60.0
tstop = tstart + self.config['orbit_period_min'] * 60.0 * self.config['gti_fraction']
exp += tstop - tstart
tstart_list.append(float(tstart))
tstop_list.append(float(tstop))
gti_list.append([tstart,tstop])
n += 1
on_source = tstop
dump = '%d events (number of events) are generated...\n' % self.config['number_of_photons']
sys.stdout.write(dump)
number_of_cycle = int((on_source)/(self.config['period']*1e-3))
sys.stdout.write('number of pulse cycle: %d (exp=%.3f ks, onSource=%.3f ks, P=%.3f ms)\n' % (
number_of_cycle,
exposure_ks, on_source/1e+3,
self.config['period']))
sys.stdout.write("[%s]" % (" " * (TOOLBAR_WIDTH+1)))
sys.stdout.flush()
sys.stdout.write("\b" * (TOOLBAR_WIDTH+2)) # return to start of line, after '['
TOOLBAR_BASE = int(self.config['number_of_photons'] / TOOLBAR_WIDTH)
#phase_list = []
#time_list = []
time_phase_list = []
# pulsed signal generation
i = 0
while i < self.config['number_of_pulsed_photons']:
phase = randomgen(func,func_range,func_max)
if self.config['time_noise_gauss_sigma'] > 0.0:
phase += numpy.random.normal(0.0,self.config['time_noise_gauss_sigma']/self.config['period'])
while phase < 0.0:
phase += 1.0
while phase > 1.0:
phase -= 1.0
event_time = (float(numpy.random.randint(0,number_of_cycle))+phase) * self.config['period']*1e-3
flag = False
for gti in gti_list:
if event_time >= gti[0] and event_time <= gti[1]:
flag = True
break
if flag == False:
continue
#phase_list.append(phase)
#time_list.append(event_time)
time_phase_list.append([event_time,phase])
i += 1
if i % TOOLBAR_BASE == 0:
sys.stdout.write("-")
sys.stdout.flush()
sys.stdout.write('\n')
# DC signal generation
i = 0
while i < self.config['number_of_DC_photons']:
phase = numpy.random.rand()
gtinum = numpy.random.randint(len(gti_list))
event_time = gti_list[gtinum][0] + phase * self.config['period']*1e-3
#phase_list.append(phase)
#time_list.append(event_time)
time_phase_list.append([event_time,phase])
i += 1
if i % TOOLBAR_BASE == 0:
sys.stdout.write("-")
sys.stdout.flush()
sys.stdout.write('\n')
time_phase_list.sort()
time_list = [ i[0] for i in time_phase_list]
phase_list = [ i[1] for i in time_phase_list]
if not os.path.exists(self.config['outdir_path']):
os.system('mkdir -p %s' % self.config['outdir_path'])
basename = '%s/%s_%dus_%dks' % (
self.config['outdir_path'],
self.config['outname'],
1000.0*float(self.config['time_noise_gauss_sigma']),
exposure_ks
)
qdpfile = '%s.qdp' % basename
fitsfile = '%s.fits' % basename
# ==========================
# FITS
# ==========================
hdu0 = pyfits.PrimaryHDU()
fits_col_TIME = pyfits.Column(name='TIME',format='1D',array=numpy.array(numpy.sort(time_list)),unit='s')
fits_coldefs = pyfits.ColDefs([fits_col_TIME])
hdu1 = pyfits.new_table(fits_coldefs)
hdu1.name = 'EVENTS'
fits_col_START = pyfits.Column(name='START',format='1D',array=numpy.array(tstart_list),unit='s')
fits_col_STOP = pyfits.Column(name='STOP',format='1D',array=numpy.array(tstop_list),unit='s')
#fits_col_START = pyfits.Column(name='START',format='1D',array=numpy.array([0.0]),unit='s')
#fits_col_STOP = pyfits.Column(name='STOP',format='1D',array=numpy.array([on_source]),unit='s')
fits_coldefs2 = pyfits.ColDefs([fits_col_START,fits_col_STOP])
hdu2 = pyfits.new_table(fits_coldefs2)
hdu2.name = 'GTI'
for header in [hdu0.header, hdu1.header, hdu2.header]:
header.append(('TIMESYS','TT',''),end=True)
header.append(('TIMEREF','LOCAL',''),end=True)
header.append(('TIMEUNIT','s',''),end=True)
header.append(('TSTART',0.0,''),end=True)
header.append(('TSTOP',on_source,''),end=True)
#header.append(('MJDREFI','51544','MJD reference day'),end=True)
#header.append(('MJDREFF','7.428703703703700E-04','MJD reference (fraction of day)'),end=True)
header.append(('TIMEDEL','1e-6','finest time resolution (time between frames'),end=True)
header.append(('TIMEPIXR','0.0','0:times refer to the beginning of bin, 0.5:midd'),end=True)
header.append(('TIMEZERO','0.0','Time Zero'),end=True)
os.system('rm -f %s' % fitsfile)
hdulist = pyfits.HDUList([hdu0, hdu1, hdu2])
hdulist.writeto(fitsfile)
#hdu = pyfits.PrimaryHDU(numpy.array(time_list))
#
#hdulist.writeto('test.fits')
self.simulated_pulse = numpy.array(numpy.histogram(phase_list,bins=self.config['number_of_bins'],range=(0.0,1.0),normed=False, weights=None)[0])
self.simulated_pulse_error = numpy.sqrt(self.simulated_pulse)
fout = open(qdpfile,'w')
fout.write('read serr 2\ncpd /xw\n')
pcofile = '%s.pco' % (os.path.splitext(qdpfile)[0])
fout.write('@%s\n' % pcofile)
for i in range(self.config['number_of_bins']):
dump = '%f %f %f \n' % (self.out_phase[i],
self.simulated_pulse[i],
self.simulated_pulse_error[i])
fout.write(dump)
# fout.write('no no no\n')
# for i in range(self.config['number_of_bins']):
# dump = '%f %f 0.0 \n' % (self.out_phase[i],
# self.offseted_pulse_profile[i]*self.config['number_of_photons']/sum(self.offseted_pulse_profile))
# fout.write(dump)
fout.close()
fout = open(pcofile,'w')
title = '%s ' % basename
title += '(P=%.3f s)' % (self.config['period']/1e+3)
ftitle = 'T=%.1f ks ' % exposure_ks
ftitle += '(On=%.1f ks) ' % (on_source/1e+3)
ftitle += 'pf=%d%% ' % (100.0*self.config['pulsed_fraction'])
ftitle += 'R=%.2f ' % (self.config['source_rate'])
ftitle += 'B=%.2f ' % (self.config['background_rate'])
ftitle += 'jit.=%d \gms ' % (1000.0*float(self.config['time_noise_gauss_sigma']))
ftitle += 'nbin=%d ' % (self.config['number_of_bins'])
# ftitle += 'pulsed:%d cnts ' % (self.config['number_of_pulsed_photons'])
dump = 'skip on\nmark on\nmark 17 on 1\nerror off 2\nline on 2\nmark off 2\n'
dump += 'la t %s\n' % title
dump += 'la f\ntime off\nlab x Phase\nlab y Counts\n'
dump += 'r x %.3f %.3f\n' % (-0.5/self.config['number_of_bins'],1+0.5/self.config['number_of_bins'])
dump += 'r y 0 %f\n' % (1.2*self.simulated_pulse.max())
#dump += 'r y 0 150\n'
#dump += 'mo gaus cons\n'
#dump += '0.5\n'
#dump += '0.01\n'
#dump += '10\n'
#dump += '20\n'
#dump += 'fit\n'
dump += 'line step on\n'
dump += 'la f %s\n' % ftitle
dump += 'la 1 P 0.0 %.1f LI 0 1 CO 4 LS 2 \n' % (self.config['number_of_background_photons']/self.config['number_of_bins'])
dump += 'lwid 3 on 2'
fout.write(dump)
fout.close()
psfile = '%s.ps' % basename
cmd = 'qdp %s <<EOF\n' % qdpfile
cmd += 'hard %s/cps\n' % psfile
cmd += 'quit\n'
cmd += 'EOF\n'
cmd += 'ps2pdf %s\n' % psfile
cmd += 'mv %s.pdf %s\n' % (os.path.splitext(os.path.basename(psfile))[0],os.path.dirname(psfile))
#cmd += 'open %s/%s.pdf \n' % (os.path.dirname(psfile),os.path.splitext(os.path.basename(psfile))[0])
print cmd; os.system(cmd)
f = open('%s.yaml' % (basename),'w')
f.write(yaml.dump(self.config))
f.close()
