def comp_stat(line, year, month, outfile):
"""
compute statistics and print them out
input: line --- command line, year, month, and output file name
command line is used by dmcopy to extract a specific location
Example: ACIS_04_2012.fits.gz[1:1024,1:256]
year --- year
month --- month
outfile --- output file name
output: outfile --- stat results in outfile
"""
cmd = ' dmcopy ' + line + ' temp.fits clobber="yes"'
expf.run_ascds(cmd)
#
#-- to avoid get min from outside of the edge of a CCD
#
cmd = ' dmimgthresh infile=temp.fits outfile=zcut.fits cut="0:1e10" value=0 clobber=yes'
expf.run_ascds(cmd)
#
#-- find avg, min, max and deviation
#
[avg, minv, minp, maxv, maxp, dev] = extract_stat_result('zcut.fits')
#
#-- find the one sigma and two sigma count rate:
#
[sigma1, sigma2, sigma3] = expf.three_sigma_values('zcut.fits')
print_stat(avg, minv, minp, maxv, maxp, dev, sigma1, sigma2, sigma3,\
year, month, outfile)
mcf.rm_files('temp.fits')
mcf.rm_files('zcut.fits')
def section_cum(ifile, erange, syear, smon, chip, s_out):
"""
create cummulative image of chip specified
input: ifile --- full ACIS image
erange --- date of the current image (e.g. 12_2018)
syear --- year of the last cummulative image
smon --- month of the last cummulative image
chip --- ccd (e.g., i2, s3)
s_out --- chip rnage (e.g. [264:1285,1416:2435] for i2)
output: <cum_acis_dir>/ACIS_07_1999_<mm>_<yyyy>_<chip>.fits.gz
<mon_acis_dir>/ACIS_<mm>_<yyy>_<chip>.fits.gz
"""
#
#--- create an image fits file for a given ccd (chip) of this month
#
sec = mon_acis_dir + 'ACIS_' + erange + '_' + chip + '.fits'
cmd = 'dmcopy ' + ifile + s_out + ' ' + sec
expf.run_ascds(cmd)
#
#--- create an cummulative image fits file for the ccd
#
last = cum_acis_dir + 'ACIS_07_1999_' + smon + '_' + syear + '_' + chip + '.fits.gz'
out = cum_acis_dir + 'ACIS_07_1999_' + erange + '_' + chip + '.fits'
cmd = 'dmimgcalc ' + last + ' ' + sec + ' ' + out + ' add'
expf.run_ascds(cmd)
#
#--- gzip the created files
#
cmd = 'gzip -f ' + sec + ' ' + out
os.system(cmd)
def find_10th(fits_file):
"""
find 10th brightest value
input: fits_file --- image fits file name
output: 10th brightest value
"""
#
#-- make histgram
#
cmd = ' dmimghist infile=' + fits_file
cmd = cmd + ' outfile=outfile.fits hist=1::1 strict=yes clobber=yes'
expf.run_ascds(cmd)
cmd = ' dmlist infile=outfile.fits outfile=./zout opt=data'
expf.run_ascds(cmd)
data = mcf.read_data_file('./zout', remove=1)
mcf.rm_files('outfile.fits')
#
#--- read bin # and its count rate
#
hbin = []
hcnt = []
for ent in data:
try:
atemp = re.split('\s+|\t+', ent)
if (len(atemp) > 3) and mcf.is_neumeric(atemp[1]) \
and mcf.is_neumeric(atemp[2]) and (int(atemp[4]) > 0):
hbin.append(float(atemp[1]))
hcnt.append(int(atemp[4]))
except:
pass
#
#--- checking 10 th bright position
#
try:
j = 0
for i in range(len(hbin) - 1, 0, -1):
if j == 9:
val = i
break
else:
#
#--- only when the value is larger than 0, record as count
#
if hcnt[i] > 0:
j += 1
return hbin[val]
except:
return 'I/INDEF'
def extract_stat_result(ifile):
"""
extract stat informaiton
input: ifile --- image fits file
output: avg --- mean
minp --- min
maxp --- max
devp --- sigma
"""
cmd = ' dmstat infile=' + ifile + ' centroid=no >' + zspace
expf.run_ascds(cmd)
data = mcf.read_data_file(zspace, remove=1)
#
#--- extract mean, dev, min, and max
#
for ent in data:
atemp = re.split('\s+|\t+', ent)
m1 = re.search('mean', ent)
m2 = re.search('min', ent)
m3 = re.search('max', ent)
m4 = re.search('sigma', ent)
if m1 is not None:
avg = atemp[1]
if m2 is not None:
minv = atemp[1]
btemp = re.split('\(', ent)
ctemp = re.split('\s+|\t+', btemp[1])
minp = '(' + ctemp[1] + ',' + ctemp[2] + ')'
if m3 is not None:
maxv = atemp[1]
btemp = re.split('\(', ent)
ctemp = re.split('\s+|\t+', btemp[1])
maxp = '(' + ctemp[1] + ',' + ctemp[2] + ')'
if m4 is not None:
dev = atemp[1]
return [avg, minv, minp, maxv, maxp, dev]
def getstat(fits):
"""
compute stat for fits image
input: fits --- fits file name
output: mean --- mean
std --- standard deviation
vmin --- min
vmax --- max
"""
cmd = 'dmstat ' + fits + ' centroid=no > ' + zspace
expf.run_ascds(cmd)
data = mcf.read_data_file(zspace, remove=1)
for ent in data:
m1 = re.search('min', ent)
m2 = re.search('max', ent)
m3 = re.search('mean', ent)
m4 = re.search('sigma', ent)
if m1 is not None:
atemp = re.split('\s+|\t+', ent)
vmin = float(atemp[1])
continue
if m2 is not None:
atemp = re.split('\s+|\t+', ent)
atemp = re.split('\s+|\t+', ent)
vmax = int(atemp[1])
continue
if m3 is not None:
atemp = re.split('\s+|\t+', ent)
mean = float(atemp[1])
continue
if m4 is not None:
atemp = re.split('\s+|\t+', ent)
std = float(atemp[1])
continue
return (mean, std, vmin, vmax)
def acis_create_cumulative(ifile='NA'):
"""
create four small section images and create cumulatvie images
input: ifile --- input file name (e.g. ACIS_05_2012.fits)
output: <cum_acis_dir>/ACIS_07_1999_<mm>_<yyyy>*.fits.gz
"""
if ifile == 'NA':
ifile = raw_input('Input file name (ACIS_<month>_<year>.fits*): ')
ifile = ifile.strip()
#
#--- find the designated date of the fits file (ifile example: ACIS_12_2018.fits)
#
atemp = re.split('.fits', ifile)
btemp = re.split('ACIS_', atemp[0])
erange = btemp[1]
ctemp = re.split('_', erange)
mon = int(ctemp[0])
year = int(ctemp[1])
#
#--- set the date of the last cummulative data files (a month before current one)
#
lmon = mon - 1
lyear = year
if lmon < 1:
lmon = 12
lyear -= 1
smon = mcf.add_leading_zero(lmon)
syear = str(lyear)
#
#--- full image
#
last = cum_acis_dir + 'ACIS_07_1999_' + smon + '_' + syear + '.fits.gz'
out = cum_acis_dir + 'ACIS_07_1999_' + erange + '.fits'
cmd = 'dmimgcalc ' + last + ' ' + ifile + ' ' + out + ' add'
expf.run_ascds(cmd)
cmd = 'gzip -f ' + out
os.system(cmd)
#
#--- CCD I2
#
section_cum(ifile, erange, syear, smon, 'i2', '[264:1285,1416:2435]')
#
#--- CCD I3
#
section_cum(ifile, erange, syear, smon, 'i3', '[1310:2331,1416:2435]')
#
#--- CCD S2
#
section_cum(ifile, erange, syear, smon, 's2', '[80:1098,56:1076]')
#
#--- CCD S3
#
section_cum(ifile, erange, syear, smon, 's3', '[1122:2141,56:1076]')
#
#--- moving this month's full image fits file to an appropriate directory
#
mc = re.search('gz', ifile)
if mc is None:
cmd = 'gzip -f ' + ifile
os.system(cmd)
mc = re.search(mon_acis_dir, ifile)
if mc is None:
cmd = 'mv ' + ifile + '* ' + mon_acis_dir + '/.'
os.system(cmd)
def comp_stat(ifile, year, month, out):
"""
compute statistics for the hrc image and print out the result
input: ifile --- hrc image file,
year --- year
month --- month
out --- output file name.
output: out
"""
if os.path.isfile(ifile):
#
#--- to avoid getting min value from the outside of the frame edge of a CCD, set threshold
#
try:
cmd = ' /bin/nice -n15 dmimgthresh infile=' + ifile
cmd = cmd + ' outfile=zcut.fits cut="0:1.e10" value=0 clobber=yes'
expf.run_ascds(cmd)
cmd = ' dmstat infile=zcut.fits centroid=no >' + zspace
expf.run_ascds(cmd)
mcf.rm_files('./zcut.fits')
data = mcf.read_data_file(zspace)
except:
data = []
val = 'NA'
for ent in data:
ent.lstrip()
m = re.search('mean', ent)
if m is not None:
atemp = re.split('\s+|\t', ent)
val = atemp[1]
break
if val != 'NA':
(mean, dev, vmin, vmax , min_pos_x, min_pos_y, max_pos_x, max_pos_y) \
= readStat(zspace)
(sig1, sig2, sig3) = expf.three_sigma_values(ifile)
else:
(mean, dev, vmin, vmax , min_pos_x, min_pos_y, max_pos_x, max_pos_y) \
= ('NA','NA','NA','NA','NA','NA','NA','NA')
(sig1, sig2, sig3) = ('NA', 'NA', 'NA')
mcf.rm_files(zspace)
else:
(mean, dev, vmin, vmax , min_pos_x, min_pos_y, max_pos_x, max_pos_y) \
= ('NA','NA','NA','NA','NA','NA','NA','NA')
(sig1, sig2, sig3) = ('NA', 'NA', 'NA')
#
#--- print out the results
#
if mean == 'NA':
line = '%d\t%d\t' % (year, month)
line = line + 'NA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\tNA\n'
else:
line = '%d\t%d\t' % (year, month)
line = line + '%5.6f\t' % (float(mean))
line = line + '%5.6f\t' % (float(dev))
line = line + '%5.1f\t' % (float(vmin))
line = line + '(%d,%d)\t' % (float(min_pos_x), float(min_pos_y))
line = line + '%5.1f\t' % (float(vmax))
line = line + '(%d,%d)\t' % (float(max_pos_x), float(max_pos_y))
line = line + '%5.1f\t' % (float(sig1))
line = line + '%5.1f\t' % (float(sig2))
line = line + '%5.1f\n' % (float(sig3))
if os.path.isfile(out):
with open(out, 'a') as f:
f.write(line)
else:
with open(out, 'w') as f:
f.write(line)
def acis_dose_get_data(startYear='', startMonth='', stopYear='', stopMonth=''):
"""
extract ACIS evt1 data from a month and create combined image file.
input: startYear --- year of starting time
startMonth --- month of starting time
stopYear --- year of stopping time
stopMonth --- month of stopping time
"""
if startYear == '' or startMonth == '' or stopYear == '' or stopMonth == '':
startYear = raw_input('Start Year: ')
startYear = int(float(startYear))
startMonth = raw_input('Start Month: ')
startMonth = int(float(startMonth))
stopYear = raw_input('Stop Year: ')
stopYear = int(float(stopYear))
stopMonth = raw_input('Stop Month: ')
stopMonth = int(float(stopMonth))
#
#--- start extracting the data for the year/month period
#
for year in range(startYear, stopYear + 1):
#
#--- create a list of month appropriate for the year
#
month_list = expf.make_month_list(year, startYear, stopYear,
startMonth, stopMonth)
for month in month_list:
smon = mcf.add_leading_zero(month)
start = str(year) + '-' + smon + '-01T00:00:00'
nextMonth = month + 1
nyear = year
if nextMonth > 12:
nextMonth = 1
nyear += 1
smon = mcf.add_leading_zero(nextMonth)
stop = str(nyear) + '-' + smon + '-01T00:00:00'
#
#--- using ar5gl, get a list of file names
#
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fitsList = mcf.run_arc5gl_process(line)
#
#--- extract each evt1 file, extract the central part, and combine them into a one file
#
for fits in fitsList:
print("FITS File: " + fits)
atemp = re.split('\s+', line)
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + fits + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
#
#--- check whether the fits file actually extracted and if so, ungip the file
#
if len(out) < 1:
continue
cmd = 'gzip -d ' + out[0]
os.system(cmd)
line = fits + '[EVENTS][bin tdetx=2800:5200:1, tdety=1650:4150:1][option type=i4]'
#
#--- create an image file
#
ichk = expf.create_image(line, 'ztemp.fits')
#
#--- combined images
#
if ichk > 0:
expf.combine_image('ztemp.fits', 'total.fits')
mcf.rm_files(fits)
mcf.rm_files('ztemp.fits')
#
#--- rename the file
#
lyear = str(startYear)
lmon = mcf.add_leading_zero(startMonth)
outfile = './ACIS_' + lmon + '_' + lyear + '_full.fits'
cmd = 'mv total.fits ' + outfile
os.system(cmd)
#
#--- trim the extreme values
#
upper = find_10th(outfile)
if mcf.is_neumeric(upper):
outfile2 = './ACIS_' + lmon + '_' + lyear + '.fits'
cmd = ' dmimgthresh infile=' + outfile + ' outfile='
cmd = cmd + outfile2 + ' cut="0:' + str(
upper) + '" value=0 clobber=yes'
expf.run_ascds(cmd)
else:
cmd = 'cp -f ' + outfile + ' ' + outfile2
os.system(cmd)
cmd = 'gzip ' + outfile
os.system(cmd)
#
#--- move full one to the data dir; keep other in <exc_dir> to be used to create cumlative files
#
cmd = 'mv ' + outfile + '* ' + mon_acis_dir + '/.'
os.system(cmd)
def createCumulative(year, month, detector, arch_dir):
"""
create cumulative hrc data for a given year and month
input: year --- year
month --- month
detector --- HRC-I or HRC-S
arch_dir --- archieve dir
output: <cum_dir>/HRC<inst>_08_1999_<mm>_<yyyy>.fits.gz
"""
#
#--- find the previous period
#
pyear = year
pmonth = month - 1
if pmonth < 1:
pmonth = 12
pyear -= 1
syear = str(year)
smonth = mcf.add_leading_zero(month)
spyear = str(pyear)
spmonth = mcf.add_leading_zero(pmonth)
if detector == 'HRC-I':
inst = 'HRCI'
else:
inst = 'HRCS'
#
#--- set file names
#
hrc = inst + '_' + smonth + '_' + syear + '.fits.gz'
chrc = inst + '_08_1999_' + spmonth + '_' + spyear + '.fits.gz'
chrc2 = inst + '_08_1999_' + smonth + '_' + syear + '.fits'
#
#--- if the monthly file exists, reduce the size of the file before combine
#--- it into a cumulative data
#
cdir = arch_dir + '/Month_hrc/'
ifile = cdir + hrc
if os.path.isfile(ifile):
line = arch_dir + '/Month_hrc/' + hrc + '[opt type=i2,null=-99]'
cmd = ' dmcopy infile="' + line + '" outfile="./ztemp.fits" clobber="yes"'
expf.run_ascds(cmd)
cmd = ' dmimgcalc infile=' + arch_dir + 'Cumulative_hrc/' + chrc
cmd = cmd + ' infile2=ztemp.fits outfile =' + chrc2 + ' operation=add clobber=yes'
expf.run_ascds(cmd)
mcf.rm_files('./ztemp.fits')
cmd = 'gzip ' + chrc2
os.system(cmd)
cmd = 'mv ' + chrc2 + '.gz ' + arch_dir + 'Cumulative_hrc/.'
os.system(cmd)
#
#--- if the monthly fie does not exist, just copy the last month's cumulative data
#
else:
try:
#test = arch_dir + 'Cumulative_hrc/' + chrc
#with open(test, 'r') as f:
# xtest = f.read()
cmd = 'cp ' + arch_dir + 'Cumulative_hrc/' + chrc + ' '
cmd = cmd + arch_dir + 'Cumulative_hrc/' + chrc2 + '.gz'
print("I AM HERE CMD: " + str(cmd))
#os.system(cmd)
except:
print("There are no last month cumulative hrc data")
subject = 'Missing HRC cumulative data'
content = 'It seems that the cummulative HRC from the last months are missing'
expf.send_warning_email(subject, content)
exit(1)