def find_start_and_stop_time():
"""
find start and stop time (set for the last month)
"""
out = time.strftime("%Y:%m", time.gmtime())
[lyear, lmon] = re.split(':', out)
tstop = lyear + ':' + lmon + ':01'
tstop = mcf.convert_date_format(tstop,
ifmt='%Y:%m:%d',
ofmt='%Y:%j:00:00:00')
year = int(float(lyear))
mon = int(float(lmon))
pyear = year
pmon = mon - 1
if pmon < 1:
pmon = 12
pyear -= 1
tstart = str(pyear) + ':' + mcf.add_leading_zero(pmon) + ':01'
tstart = mcf.convert_date_format(tstart,
ifmt='%Y:%m:%d',
ofmt='%Y:%j:00:00:00')
return [tstart, tstop]
def create_date_list_to_yestaday(testfits, yesterday=''):
"""
find the last entry date and then make a list of dates up to yesterday
input: testfits --- a fits file to be tested
yesterday --- date of yesterday in the format of yyyymmdd
output: otime --- a list of date in the format of yyyymmdd
"""
try:
test = float(yesterday)
chk = 1
except:
chk = 0
if chk == 0:
out = time.strftime('%Y:%j:00:00:00', time.gmtime())
yesterday = Chandra.Time.DateTime(out).secs - 86400.0
#
ltime = find_the_last_entry_time(testfits)
out = mcf.convert_date_format(ltime, ifmt='chandra', ofmt='%Y:%j:00:00:00')
out = Chandra.Time.DateTime(out).secs
t_list = [out]
ntime = out + 86400.0
while ntime <= yesterday:
t_list.append(ntime)
ntime += 87400.0
otime = []
for ent in t_list:
out = mcf.convert_date_format(ent, ifmt='chandra', ofmt='%Y%m%d')
otime.append(out)
return otime
def check_time_format(intime):
"""
return time in Chandra time
input: intime --- time in <yyyy>:<ddd>:<hh>:<mm>:<ss> or <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or chandra time
output: time in chandra time (seconds from 1998.1.1)
"""
mc1 = re.search('-', intime)
mc2 = re.search(':', intime)
#
#--- it is already chandra format
#
if mcf.chkNumeric(intime):
return int(float(intime))
#
#--- time in <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
elif mc1 is not None:
mc2 = re.search('T', intime)
if mc2 is not None:
stime = mcf.convert_date_format(intime,
ifmt='%Y-%m-%d:%H:%M:%S',
ofmt='chandra')
else:
stime = mcf.convert_date_format(intime,
ifmt='%Y-%m-%d',
ofmt='chandra')
return stime
#
#--- time in <yyyy>:<ddd>:<hh>:<mm>:<ss>
#
elif mc2 is not None:
return Chandra.Time.DateTime(intime).secs
def extract_sim_position(year, period_start, period_end):
"""
extract sim position information from comprehensive_data_summary data file
input: year --- year (in form of 2012)
period_start --- start time in seconds from 1.1.1998
period_end --- stop time in seconds from 1.1.1998
output: time --- (seconds from 1.1.1998)
sim_position
"""
sim_time = []
sim_pos = []
ifile = mj_dir + '/comprehensive_data_summary' + str(year)
data = mcf.read_data_file(ifile)
for ent in data:
atemp = re.split('\s+|\t+', ent)
try:
tinsec = mcf.convert_date_format(atemp[0], ofmt='chandra')
except:
continue
if tinsec >= period_start and tinsec < period_end:
sim_time.append(float(tinsec))
sim_pos.append(float(atemp[1]))
return [sim_time, sim_pos]
def acis_sci_run_print_html(html_dir, pyear, pmonth, pday):
"""
update three html pages according for the year (pyear)
Input: html_dir --- web directory path
pyear --- the year you want to update the html pages
pmonth --- current month
pday --- current month date
Output: science_run.html
science_run<year>.html
"""
#
#--- set substitution values
#
pyear = int(pyear)
dtime = str(pyear) + ':' + str(pmonth) + ':' + str(pday) + ':00:00:00'
ydate = int(
float(
mcf.convert_date_format(dtime, ifmt='%Y:%m:%d:%H:%M:%S',
ofmt='%j')))
update = str(pyear) + '-' + str(pmonth) + '-' + str(pday)
#
#--- make a link table
#
ylist = ''
j = 0
for ryear in range(1999, pyear + 1):
ylist = ylist + '<td><a href=' + http_dir + '/Year' + str(ryear)
ylist = ylist + '/science_run' + str(ryear) + '.html>'
ylist = ylist + '<strong>Year ' + str(
ryear) + '</strong></a><br /><td />\n'
#
#--- every 6 years, break a row
#
if j == 5:
j = 0
ylist = ylist + '</tr><tr>\n'
else:
j += 1
#
#---- update the main html page
#
template = house_keeping + 'science_run.html'
outfile = html_dir + 'science_run.html'
print_html_page(template, update, pyear, ylist, outfile)
#
#--- update sub directory html pages
#
ystop = pyear + 1
for syear in range(1999, ystop):
template = house_keeping + 'sub_year.html'
outfile = html_dir + 'Year' + str(syear) + '/science_run' + str(
syear) + '.html'
if syear == pyear:
ldate = update
else:
ldate = str(syear) + '-12-31'
print_html_page(template, ldate, syear, ylist, outfile)
def check_date():
"""
check wether there is an output directory and if it is not, create one
input: none
output: year --- the current year
mon --- the current month
mdir --- the current output direcotry
"""
#
#--- find today's date
#
out = time.strftime('%Y:%j:00:00:00', time.gmtime())
stime = Chandra.Time.DateTime(out).secs)
#
#--- find 10 days ago and create the output directory based on that date
#
s10 = stime - 10 * 86400.0
out = mcf.convert_date_format(s10, ifmt='chandra', ofmt='%Y:%m:%d')
atemp = re.split(':', out)
year = int(float(atemp[0]))
mon = int(float(atemp[1]))
lmon = mcf.change_month_format(mon)
mdir = web_dir + lmon.upper() + str(year)
if not os.path.isdir(mdir)
cmd = 'mkdir -p ' + mfile
os.system(cmd)
return (year, mon, mdir)
def find_data_collection_interval():
"""
find data collection period in dom
input: none but read from <data_dir>/Dis_dir/hist_ccd3
output: ldom --- starting dom
tdom --- stopping dom (today)
"""
#
#--- find today's date
#
tout = time.strftime('%Y:%j:%H:%M:%S', time.gmtime())
tdate = int(mcf.convert_date_format(tout, ofmt='chandra'))
#
#--- find the date of the last entry
#
ifile = data_dir + 'hist_ccd3'
data = mcf.read_data_file(ifile)
data.reverse()
for ent in data:
atemp = re.split('<>', ent)
try:
ldate = int(float(atemp[0]))
break
except:
continue
#
#--- the data colleciton starts from the next day of the last entry date
#
ldate += 86400
return [tdate, ldate]
def read_data_info(year):
"""
read data info data for a given year
input: year --- the year of the data set
output: [time, sim_x, sim_y, sim_z, pitch, yaw]
"""
infile = data_dir + 'data_info_' + str(year)
data = mcf.read_data_file(infile)
time = []
sim_x = []
sim_y = []
sim_z = []
pitch = []
yaw = []
for ent in data:
atemp = re.split('\s+', ent)
out = mcf.convert_date_format(atemp[3],
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='chandra')
start = mcf.chandratime_to_fraq_year(out)
out = mcf.convert_date_format(atemp[4],
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='chandra')
stop = mcf.chandratime_to_fraq_year(out)
mid = 0.5 * (start + stop)
try:
val1 = float(atemp[5])
val2 = float(atemp[6])
val3 = float(atemp[7])
val4 = float(atemp[8]) * 3600.0
val5 = float(atemp[9]) * 3600.0
time.append(mid)
sim_x.append(val1)
sim_y.append(val2)
sim_z.append(val3)
pitch.append(val4)
yaw.append(val5)
except:
continue
return [time, sim_x, sim_y, sim_z, pitch, yaw]
def dom_to_stime(dom):
"""
convert dom into seconds from 1998.1.1
input: dom --- dom (day of mission)
output: stime --- seconds from 1998.1.1
"""
[year, ydate] = mcf.dom_to_ydate(dom)
etime = str(year) + ':' + str(ydate)
stime = mcf.convert_date_format(etime, ifmt='%Y:%j', ofmt='chandra')
return stime
def covertfrom1998sec(stime):
"""
convert second from 1998.1.1 to yyyy-mm-ddThh:mm:ss format
input: stime --- second from 1998.1.1
output: etime --- time in yyyy-mm-ddThh:mm:ss
"""
etime = mcf.convert_date_format(stime,
ifmt='chandra',
ofmt='%Y-%m-%dT%H:%M:%S')
return etime
def chandratime_to_yday(ctime):
"""
convert chandra time into a day of year
input: ctime --- time in seconds from 1998.1.1
output: ydate --- a day of year (fractional)
"""
atime = mcf.convert_date_format(ctime, ofmt='%Y:%j:%H:%M:%S')
btemp = re.split(':', atime)
year = float(btemp[0])
ydate = float(btemp[1])
hour = float(btemp[2])
mins = float(btemp[3])
sec = float(btemp[4])
ydate = ydate + (hour / 24.0 + mins / 1440.0 + sec / 86400.0)
return [year, ydate]
def get_data_from_db(obsid):
"""
extract observation information from the database
input: obsid --- obsid
output: tsec --- the data of the observation in seconds from 1998.1.1
line --- a string of the information extract
<obsid> <target name> <obs date> <obs date in sec>
<target id> < sequence number>
"""
#
#-- sql command
#
cmd = 'select targname,instrument,soe_st_sched_date,targid,'
cmd = cmd + 'seq_nbr from target where obsid=' + str(obsid)
try:
#
#--- call sql database
#
out = sser.set_sybase_env_and_run(cmd, fetch='fetchone')
#
#--- output is a list of the data
#
target = clean_name(out[0])
inst = out[1]
odate = out[2]
targid = out[3]
seqno = out[4]
#
#--- convert time into sec from 1998.1.1
#
tsec = mcf.convert_date_format(odate,
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='chandra')
line = str(obsid) + '\t' + target + '\t' + str(odate) + '\t' + str(
tsec)
line = line + '\t' + str(targid) + '\t' + str(seqno) + '\n'
return [tsec, line]
except:
return NULL
def get_last_entry_time(infile, pos=4):
"""
find the last entry date from the database
input: infile --- input file, ususally data_infor_<year> and assume that
pos-th entry is the (ending) time
pos --- position of the time element in the data (usually 4th)
output: ftime --- the last entry data in <yyy>:<ddd>:<hh>:<mm>:<ss>
"""
data = mcf.read_data_file(infile)
if len(data) == 0:
print("something wrong in starting time; exist")
exit(1)
else:
atemp = re.split('\s+', data[-1])
ltime = atemp[pos]
#
#--- convert time format to be used
#
ftime = mcf.convert_date_format(ltime, ifmt='%Y-%m-%dT%H:%M:%S', ofmt='%Y:%j:%H:%M:%S')
return ftime
def create_display_data_table():
"""
create a readable data table for html page
Input: none, but read from <data_dir>/ccd<ccd>_<node>
Output: <web_dir>/ccd<ccd>_<node>
"""
for ccd in range(0, 10):
for node in range(0, 4):
ifile = 'ccd' + str(ccd) + '_' + str(node)
#
#--- read the original data file
#
infile = data_dir + ifile
data = mcf.read_data_file(infile)
#
#--- adding heading
#
line = "#\n#Date Mn K alpha Al K alpha "
line = line + "Ti K alpha Slope Sigma Int Sigma\n#\n"
for ent in data:
atemp = re.split('\s+', ent)
#
#--- converting the date format from chandra time into <mon> <year> for html data display
#
stime = int(atemp[0])
out = mcf.convert_date_format(stime,
ifmt='chandra',
ofmt='%Y:%m:%d')
atemp = re.split(':', out)
lmon = mcf.change_month_format(atemp[1])
ldate = lmon + ' ' + atemp[0]
lout = ent.replace(atemp[0], ldate)
line = line + lout + '\n'
outfile = web_dir + 'Data/' + ifile
with open(outfile, 'w') as fo:
fo.write(line)
def find_starting_date():
"""
set starting and stopping time from the last entry of a fits file
input: none but read from hvpsstat_data.fits
output: start --- starting time <yyyy>-<mm>-<dd>T00:00:00
stop --- stoping time <yyyy>-<mm>-<dd>T23:59:59
"""
test = dpath + 'hvpsstat_data.fits'
ltime = ecf.find_the_last_entry_time(test) + 86400.0
ltime = mcf.convert_date_format(ltime, ifmt='chandra', ofmt='%Y:%j:00:00:00')
ltime = Chandra.Time.DateTime(ltime).secs
today = time.strftime('%Y:%j:00:00:00', time.gmtime())
stime = Chandra.Time.DateTime(today).secs - 86400.0
t_list = [ltime]
while ltime < stime:
ltime += 86400.0
if ltime > stime:
break
else:
t_list.append(ltime)
return t_list
def convert_stime_to_ytime(stime):
"""
convert a list of time in seconds to fractional year
input: stime --- a list of time in seconds from 1998.1.1
output: ytime --- a list of time in fractional year
"""
ytime = []
for ent in stime:
out = mcf.convert_date_format(ent, ofmt="%Y:%j:%H:%M:%S")
atemp = re.split(':', out)
year = float(atemp[0])
yday = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
if mcf.is_leapyear(year):
base = 366.0
else:
base = 365.0
year += (yday + hh / 24.0 + mm / 1440.0 + ss / 86400.) / base
ytime.append(year)
return ytime
def alignment_sim_twist_extract(tstart='', tstop='', year=''):
"""
extract sim twist data and update the databases
input: tstart --- starting time in the format of <yyyy>:<ddd>:<hh>:<mm>:<ss>
default: "" --- the script will find the starting time
tstop --- stopping time in the format of <yyyy>:<ddd>:<hh>:<mm>:<ss>
default: "" --- the script will find the stopping time
year --- year, default: "".
output: <data_dir>/data_info_<year>
<data_dir>/data_extracted_<year>
"""
#
#--- if the starting and stopping time are given, use them
#
if tstart != '':
ending = extract_data(tstart, tstop, year, ldir='./')
#
#--- otherwise, find stating and stopping time
#
else:
#
#--- current time
#
today = time.strftime("%Y:%j:%H:%M:00", time.gmtime())
year = int(float(time.gmtime().tm_year))
#
#--- find the last entry time
#
ifile = data_dir + 'data_info_' + str(year)
if os.path.isfile(ifile):
out = mcf.read_data_file(ifile)
data = []
for ent in out:
if ent != '':
data.append(ent)
else:
#
#--- if the current year data files are not there, just creates an empty list
#
data = []
#
#--- if this is a new year, read the last year's data
#
if len(data) == 0:
ifile2 = data_dir + 'data_info_' + str(year - 1)
tstart = get_last_entry_time(ifile2)
tstop = str(year) + ':001:00:00:00'
ending = extract_data(tstart, tstop, year - 1)
#
#--- check the last entry again for the next round (for the new year data set)
#
tstart = mcf.convert_date_format(ending,
ifmt='%Y-%m-%dT%H:%M:%S',
ofmt='%Y:%j:%H:%M:%S')
#
#--- work on this year's data
#
else:
tstart = get_last_entry_time(ifile)
ending = extract_data(tstart, today, year)
def report_error():
"""
read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email
Input: none but read from <cup_usr_list>_error_list
Output: email sent out
"""
#
#--- find the current time
#
out = time.strftime('%Y:%m:%d', time.gmtime())
[year, mon, day] = re.split(':', out)
#
#--- set cutting date for the report
#
out = time.strftime('%Y:%j:%H:%M:%S', time.gmtime())
cut = Chandra.Time.DateTime(out).secs - 1.5 * 86400.0
#
#--- create surfix for files which will be saved in Past_errors directory
#
smon = mcf.add_leading_zero(mon)
sday = mcf.add_leading_zero(day, dlen=3)
tail = str(year) + smon + sday
for tag in cpu_usr_list:
efile = house_keeping + 'Records/' + tag + '_error_list'
pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail
prev_line = ''
if os.path.isfile(efile):
#
#--- read error messages from the file
#
data = mcf.read_data_file(efile)
#
#--- sort the data so that we can correct messages to each cron job together
#
data.sort()
task_list = []
time_list = []
mssg_list = []
for ent in data:
atemp = re.split(' : ' , ent)
otime = int(float(atemp[1]))
dtime = mcf.convert_date_format(str(otime), ifmt='%Y%m%d%H%M%S', ofmt='%Y:%j:%H:%M:%S')
#
#--- if the error is more than <cut> day old, ignore
#
stime = Chandra.Time.DateTime(dtime).secs
if stime < cut:
continue
task_list.append(atemp[0])
time_list.append(dtime)
mssg_list.append(atemp[2])
#
#--- write out cron job name
#
cname = task_list[0]
sline = '\n\n' + cname + '\n____________________\n\n'
for i in range(1, len(mssg_list)):
if task_list[i] != cname:
cname = task_list[i]
sline = sline + '\n\n' + cname + '\n____________________\n\n'
#
#--- create each line. if it is exactly same as one line before, skip it
#
line = time_list[i] + ' : ' + mssg_list[i] + '\n'
if line != prev_line:
sline = sline + line
prev_line = line
with open(zspace, 'w') as fo:
fo.write(sline)
#
#--- send email out
#
send_mail(tag, email_list)
#
#--- move the error list to Past_errors directory
#
if os.path.isfile(efile): #--- 03/06/19
cmd = 'mv ' + efile + ' ' + pfile
os.system(cmd)
def report_about_data(r1):
"""
reporting the general ephemeris information
input: r1 --- the dictionary of ephemeris information
output: screen print out of the information
"""
line = ''
dval = str(r1['start_date'])
mc = re.search('\.', dval)
if mc is not None:
atemp = re.split('\.', dval)
dval = atemp[0]
out = mcf.convert_date_format(dval, ifmt='%y%m%d', ofmt='%Y:%m:%d')
[year, mon, day] = re.split(':', out)
line = line + 'Date of first ephemeris point (Year, Month, Day): '
line = line + year + ' ' + mon + ' ' + day + '\n'
#
#--- convert date format:e.g., 190505.0 --> 20190505.0
#
if r1['start_date'] < 500000:
r1['start_date'] += 20000000
else:
r1['start_date'] += 19000000
if r1['stop_date'] < 500000:
r1['stop_date'] += 20000000
else:
r1['stop_date'] += 19000000
if r1['epoch_year'] < 50:
r1['epoch_year'] += 2000
else:
r1['epoch_year'] += 1900
line = line + "Start/Stop/Epoc Date: " + str(
r1['start_date']) + ' : ' + str(r1['stop_date'])
line = line + ' : ' + str(r1['epoch_year']) + '\n'
line = line + '\nChandra Keplerian Elements: \n'
line = line + 'Elements Epoch: \n'
line = line + ' (YYYY MM DD hh mm ss.sss):' + str(int(
r1['epoch_year'])) + ', '
line = line + str(int(r1['epoch_month'])) + ', ' + str(int(
r1['epoch_day'])) + ', '
line = line + str(int(r1['epoch_hour'])) + ', ' + str(int(
r1['epoch_min'])) + ', '
line = line + str(int(r1['epoch_millisec'] / 1000.0)) + '.'
line = line + str(int(r1['epoch_millisec'] % 1000))
line = line + '\n'
line = line + ' (Seconds since start of 1985): %16.3f\n' % r1[
'epoch_time']
line = line + '\n\n'
line = line + ' Semi-major axis (km): ' + str(
r1['smajor_axis']) + '\n'
line = line + ' Eccentricity: ' + str(
r1['eccent']) + '\n'
line = line + ' Inclination (deg): ' + str(
r1['inclin'] * r2d) + '\n'
line = line + ' Argument of Perigee (deg): ' + str(
r1['perigee'] * r2d) + '\n'
line = line + ' RA of Ascending Node (deg): ' + str(
r1['raan'] * r2d) + '\n'
line = line + ' Mean Anomaly (deg): ' + str(
r1['mean_anom'] * r2d) + '\n'
line = line + ' True Anomaly (deg): ' + str(
r1['true_anom'] * r2d) + '\n'
line = line + ' Arg of Perigee + True Anom (deg):' + str(
r1['sum_aprgta'] * r2d) + '\n'
line = line + ' (approx.) Period (hours): ' + str(
24 * r1['period'] / 100) + '\n'
line = line + ' (approx.) Mean Motion (deg/hour):'
line = line + str(r1['mean_motn'] * r2d * 100 / 24) + '\n'
line = line + ' (wrong) Rate of change of RA of AN (deg/day):'
line = line + str(r1['rate_ascnd'] * r2d * 100.0) + '\n'
print(line)
def define_x_range(dlist, xunit=''):
"""
set time plotting range
Input: dlist --- list of data files (e.g., Data_2012_09)
Output: start --- starting time in either ydate or fractional year
end --- ending time in either ydate or fractional year
"""
num = len(dlist)
if num == 1:
atemp = re.split('\/', dlist[0])
btemp = re.split('Data_', atemp[-1])
ctemp = re.split('_', btemp[1])
year = int(ctemp[0])
month = int(ctemp[1])
nyear = year
nmonth = month + 1
if nmonth > 12:
nmonth = 1
nyear += 1
else:
slist = sorted(dlist)
atemp = re.split('\/', slist[0])
btemp = re.split('Data_', atemp[-1])
ctemp = re.split('_', btemp[1])
year = int(ctemp[0])
month = int(ctemp[1])
atemp = re.split('\/', slist[len(slist)-1])
btemp = re.split('Data_', atemp[-1])
ctemp = re.split('_', btemp[1])
tyear = int(ctemp[0])
tmonth = int(ctemp[1])
nyear = tyear
nmonth = tmonth + 1
if nmonth > 12:
nmonth = 1
nyear += 1
#
#--- get day of year
#
start = str(year) + ':' + mcf.add_leading_zero(month) + ':01'
stop = str(nyear) + ':' + mcf.add_leading_zero(nmonth) + ':01'
start = int(mcf.convert_date_format(start, ifmt='%Y:%m:%d', ofmt='%j'))
stop = int(mcf.convert_date_format(stop, ifmt='%Y:%m:%d', ofmt='%j'))
#
#--- for te case it is a month plot
#
if nyear > year:
if mcf.is_leapyear(year):
end = stop + 366
else:
end = stop + 365
else:
end = stop
#
#--- for the case it is a longer plot
#
if xunit == 'year':
if mcf.is_leapyear(year):
base = 366.0
else:
base = 365.0
start = year + float(start) / base
if mcf.is_leapyear(nyear):
base = 366.0
else:
base = 365.0
end = year + float(stop) / base
return [start, end]
def acis_hist_extract_data(year, month):
"""
extract acis histgram data
input: year --- year
month --- month
output: <data_dir>/ccd<ccd>_node<node>_<lpos>
"""
#
#--- clean up and/or create working directries
#
if os.path.isdir(exc_dir):
cmd = 'rm -f ' + exc_dir + '*.fits* 2>/dev/null'
else:
cmd = 'mkdir -p ' + exc_dir
os.system(cmd)
t_dir = exc_dir + 'Temp_dir'
if os.path.isdir(t_dir):
cmd = 'rm -f ' + t_dir + '/* 2>/dev/null'
else:
cmd = 'mkdir -p ' + t_dir
os.system(cmd)
#
#--- define extracting period in seconds from 1.1.1998
#
end_year = year
end_month = month + 1
if end_month > 12:
end_month = 1
end_year += 1
cmon = str(month)
if month < 10:
cmon = '0' + cmon
sdate = str(year) + ':' + cmon + ':01:00:00:00'
pstart = mcf.convert_date_format(sdate, '%Y:%m:%d:%H:%M:%S', 'chandra')
cmon = str(end_month)
if end_month < 10:
cmon = '0' + cmon
pdate = str(end_year) + ':' + cmon + ':01:00:00:00'
pend = mcf.convert_date_format(pdate, '%Y:%m:%d:%H:%M:%S', 'chandra')
mdate = 0.5 * (pstart + pend)
#
#--- extract sim position information
#
twoarray = extract_sim_position(year, pstart, pend)
sim_time = twoarray[0]
sim_pos = twoarray[1]
#
#----extract acis hist data from archive using arc5gl
#
data_list = use_arc5gl_acis_hist(year, month, end_year, end_month)
#
#--- create data array: histogram row: 4000, ccd #: 10, node #: 4, position: 0, 1
#
hdata = numpy.zeros((4000, 10, 4, 2))
duration = numpy.zeros((10, 4, 2))
#
#--- go though each extracted acis hist data
#
for ifile in data_list:
atemp = re.split('\/', ifile)
fname = atemp[len(atemp) - 1]
fname = fname.replace('.fits', '')
#
#--- extract head info
#
ccd_info = extract_head_info(ifile)
[fep, ccd, node, pblock, tstart, tstop, expcount, date_obs,
date_end] = ccd_info
tdiff = float(tstop) - float(tstart)
ccd = int(ccd)
#
#--- we need to check only ccd = 1, 2, 3, 6, and 7
#
if ccd in [0, 4, 5, 8, 9]:
mcf.rm_files(ifile)
continue
pblock = int(pblock)
tstart = int(tstart)
tstop = int(tstop)
if pblock in [2334740, 2342932, 2334756, 2342948]:
pass
else:
continue
#
#--- find average sim position
#
sim_info = find_sim_position(sim_time, sim_pos, tstart, tstop)
[sim_avg, sim_min, sim_max] = sim_info
#
#--- for the case that the sim is at the external calibration source position
#
if (sim_min > -100800 and sim_min < -98400) and (sim_max > -100800
and sim_max < -98400):
#
#
#--- pblock values were changed in year 2012 (2012.1.1 == 441763197). ---------
#
if float(tstart) < 441763197:
#
#--- ccd rows are between 801 and 1001 before Year 2012
#
if pblock == 2334740:
hist_data = extract_hist_data(ifile)
duration[ccd][node][1] += tdiff
for i in range(0, 4000):
hdata[i][ccd][node][1] += hist_data[i]
#
#--- ccd rows are between 21 and 221 before Year 2012
#
elif pblock == 2342932:
hist_data = extract_hist_data(ifile)
duration[ccd][node][0] += tdiff
for i in range(0, 4000):
hdata[i][ccd][node][0] += hist_data[i]
#
#--- after 2012.1.1 ---------
#
else:
#
#--- ccd rows are between 801 and 1001 from Year 2012
#
if pblock == 2334756:
hist_data = extract_hist_data(ifile)
duration[ccd][node][1] += tdiff
for i in range(0, 4000):
hdata[i][ccd][node][1] += hist_data[i]
#
#--- ccd rows are between 21 and 221 from Year 2012
#
elif pblock == 2342948:
hist_data = extract_hist_data(ifile)
duration[ccd][node][0] += tdiff
for i in range(0, 4000):
hdata[i][ccd][node][0] += hist_data[i]
mcf.rm_files(ifile)
#
#--- now fit the parameters for a combined data set
#
test_out = []
for pos in (0, 1):
for ccd in (1, 2, 3, 6, 7):
for node in range(0, 4):
darray = [0 for x in range(0, 4000)]
sum = 0
for i in range(0, 4000):
darray[i] = hdata[i][ccd][node][pos]
sum += hdata[i][ccd][node][pos]
if sum > 10:
peak_info = find_peaks(darray)
if pos == 0:
lpos = 'low'
else:
lpos = 'high'
ifile = data_dir + 'ccd' + str(ccd) + '_node' + str(
node) + '_' + lpos
with open(ifile, 'a') as fo:
line = str(year) + ':' + str(month) + '\t' + str(
mdate) + '\t'
line = line + str(peak_info[0]) + '\t' + str(
peak_info[9]) + '\t'
line = line + str(peak_info[1]) + '\t' + str(
peak_info[10]) + '\t'
line = line + str(peak_info[2]) + '\t' + str(
peak_info[11]) + '\t'
line = line + str(peak_info[3]) + '\t' + str(
peak_info[12]) + '\t'
line = line + str(peak_info[4]) + '\t' + str(
peak_info[13]) + '\t'
line = line + str(peak_info[5]) + '\t' + str(
peak_info[14]) + '\t'
line = line + str(peak_info[6]) + '\t' + str(
peak_info[15]) + '\t'
line = line + str(peak_info[7]) + '\t' + str(
peak_info[16]) + '\t'
line = line + str(peak_info[8]) + '\t' + str(
peak_info[17]) + '\t'
line = line + str(duration[ccd][node][pos]) + '\n'
fo.write(line)
lmonth = str(month)
if month < 10:
lmonth = '0' + lmonth
fname = 'hist_' + str(year) + '_' + lmonth
eparam = peak_info[0:9]
plot_fit(darray, eparam, fname, ccd, node, lpos)
def get_data_for_day(year, month, day=''):
"""
extract one month amount of data and update data files
input: year --- year of the data
month --- month of the data
output: updated data files:
<MMM><YYYY>/ccd#
<MMM><YYYY>/ephin_data
Note: there is no ephin data after 2018 Nov
"""
#
#--- if the date is given...
#
chk = 0
if year != '':
lmon = mcf.add_leading_zero(month)
lday = mcf.add_leading_zero(day)
if day != '':
start = str(year) + ':' + lmon + ':' + lday + ':00:00:00'
stop = str(year) + ':' + lmon + ':' + lday + ':23:59:59'
#
#-- if the day part is not given, compute for the entire month
#
else:
nyear = year
nmon = mon + 1
if nmon > 12:
nyear += 1
nmon = 1
nmon = mcf.add_leading_zero(nmon)
start = str(year) + ':' + lmon + ':' + ':01:00:00:00'
stop = str(nyear) + ':' + nmom + ':' + ':01:00:00:00'
chk = 1
start = mcf.convert_date_format(start,
ifmt='%Y:%m:%d:%H:%M:%S',
ofmt='chandra')
stop = mcf.convert_date_format(stop,
ifmt='%Y:%m:%d:%H:%M:%S',
ofmt='chandra')
#
#--- if the year/month are not given, extract data of the day before
#
if year == '':
out = time.strftime('%Y:%j:00:00:00', time.gmtime())
stop = Chandra.Time.DateTime(out).secs
start = stop - 86400.0
out = mcf.convert_date_format(start, ifmt='chandra', ofmt='%Y:%m')
out = re.split(':', out)
year = int(float(out[0]))
month = int(float(out[1]))
#
#--- set output directory name
#
cmonth = mcf.change_month_format(month) #--- convert digit to letter month
ucmon = cmonth.upper()
dir_name = data_dir + '/' + ucmon + str(year) + '/' #--- output directory
if not os.path.isdir(dir_name):
cmd = 'mkdir -p ' + dir_name
os.system(cmd)
#
#--- if the entire month data needs to be extracted, remove all previous data
#
if chk == 1:
cmd = 'rm -rf ' + dir_name + '/ccd*'
os.system(cmd)
#
#--- get acis count rate data
#
extract_acis_count_rate(start, stop, dir_name)
#
#--- get ephin rate data; no data after Nov 2018
#
if year < 2018:
get_ephin_data(start, stop, dir_name)
elif (year == 2018) and (month < 11):
get_ephin_data(start, stop, dir_name)
#
#-- clean the data files
#
cleanUp(dir_name)
def create_adjusted_cti_tables(ccd_list, sub_dir, sub2_dir_list, out_dir):
"""
create adjusted cti table: Data119, Data2000, Data7000, Data_cat-adjust
Input: ccd_list --- a list of ccd #
sub_dir --- a name of sub directory which cti lists are read
sub2_dir_list --- a list of sub directories which adjusted cti tables are deposited
out_dir --- a directory name which adjucted cti will be deposited
cti data are read from <data_dir>/<sub_dir>/<elm>_ccd<ccd #>
Output: <data_dir>/<dir in sub2_dir_list>/<elm>_ccd<ccd#>
"""
for elm in elm_list:
for ccd in ccd_list:
if ccd in [5, 7]:
factor = 0.045 #--- these factors are given by C. Grant
else:
factor = 0.036
#
#--- read the main data set
#
ifile = data_dir + sub_dir + '/' + elm + '_ccd' + str(ccd)
data = mcf.read_data_file(ifile)
save1 = []
save2 = []
save3 = []
save4 = []
time3 = []
time4 = []
del_temp = []
for ent in data:
#
#--- convert time format from 2013-10-05T17:35:58 to year, stime and fractional year
#
atemp = re.split('\s+', ent)
sectime = mcf.convert_date_format(atemp[0], ifmt="%Y-%m-%dT%H:%M:%S", ofmt='chandra')
fyear = mcf.chandratime_to_fraq_year(sectime)
tspan = int(atemp[12]) - int(atemp[11])
temperature = float(atemp[7])
#
#--- use only data with the integration time longer than 1000 sec before 2003
#--- and only data with the integration time longer than 2000 sec after 2003
#
if tspan < 1000:
pass
elif (tspan < 2000) and (fyear >= 2003):
pass
line = ent + '\n'
save3.append(line)
time3.append(fyear)
#
#--- we need to adjust focal plane temperature between 9/16/2005 - 10/16/2005
#--- a reported temperature is about 1.3 warmer than a real focal temperature
#--- (from 12/1/05 email from C. Grant)
#
if (sectime >= 243215999) and (sectime <= 245894399):
temperature -= 1.3
if temperature <= -119.5:
line = ent + '\n'
save1.append(line)
if tspan >= 7000:
save4.append(line)
time4.append(fyear)
#
#--- correct temperature dependency with C. Grat factors
#
val = factor * (temperature + 119.87)
quad0 = select_grant_cti(atemp[1], val)
quad1 = select_grant_cti(atemp[2], val)
quad2 = select_grant_cti(atemp[3], val)
quad3 = select_grant_cti(atemp[4], val)
if (quad0 != 'na') and (quad1 != 'na') and (quad2 != 'na') and (quad3 != 'na'):
line = atemp[0] + '\t'
line = line + quad0 + '\t'
line = line + quad1 + '\t'
line = line + quad2 + '\t'
line = line + quad3 + '\t'
line = line + atemp[5] + '\t'
line = line + atemp[6] + '\t'
line = line + atemp[7] + '\t'
line = line + atemp[8] + '\t'
line = line + atemp[9] + '\t'
line = line + atemp[10] + '\t'
line = line + atemp[11] + '\t'
line = line + atemp[12] + '\n'
save2.append(line)
#
#--- print out adjsted cti data table
#
j = 0
for sdir in sub2_dir_list:
j += 1
sdata = eval('save%s' % (j))
print_cti_results(sdir, elm, ccd, sdata)
#
#---- compute adjusted cti values and update tables
#
compute_adjusted_cti(elm, ccd, time3, save3, time4, save4, out_dir)
def create_phase_list():
"""
extract the phase numbers from mit site and creates a list
input: none but read from MIT site: http://acis.mit.edu/asc/
output: phase_list --- a list of the phase numbers
start_list --- a list of the phase starting time in seconds from 1998.1.1
end_list --- a list of the phase ending time in seconds from 1998.1.1
"""
#
#--- using lynx, read the mit web page where shows phase list
#
cmd = '/usr/bin/lynx -source http://acis.mit.edu/asc/ >' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
phase_list = []
start_list = []
end_list = []
chk = 0
for ent in data:
if chk == 0:
mc = re.search('nbsp;Phase', ent)
if mc is not None:
atemp = re.split('Phase', ent)
btemp = re.split('\<', atemp[1])
val = float(btemp[0])
phase_list.append(int(val))
chk = 1
continue
elif chk == 1:
atemp = re.split('tt\>', ent)
btemp = re.split('\<', atemp[1])
ltime = btemp[0]
try:
stime = int(
mcf.convert_date_format(ltime,
ifmt='%m/%d/%y',
ofmt='chandra'))
start_list.append(stime)
chk = 2
continue
except:
continue
elif chk == 2:
try:
atemp = re.split('tt\>', ent)
btemp = re.split('\<', atemp[1])
ltime = btemp[0]
except:
end_list.append(
6374591994) #--- the latest period has an open ending
chk = 0
continue
try:
stime = int(
mcf.convert_date_format(ltime,
ifmt='%m/%d/%y',
ofmt='chandra'))
end_list.append(stime)
if val == 1:
break
chk = 0
continue
except:
continue
#
#--- reverse all list to the oldest to newest
#
phase_list = list(reversed(phase_list))
start_list = list(reversed(start_list))
end_list = list(reversed(end_list))
return [phase_list, start_list, end_list]
