def convert_to_fyear(cdate):
"""
convert time format to fractional year
input: cdate --- time in either <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> or seconds from 1998.1.1
output: fyear --- time in fractional year
"""
try:
mc = re.search('T', cdate)
except:
mc = None
#
#--- for the case the time format is: <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
#
if mc is not None:
atemp = re.split('T', cdate)
btemp = re.split('-', atemp[0])
ctemp = re.split(':', atemp[1])
year = float(btemp[0])
mon = float(btemp[1])
day = float(btemp[2])
hh = float(ctemp[0])
mm = float(ctemp[1])
ss = float(ctemp[2])
ydate = mon_list[int(mon) - 1] + day
if tcnv.isLeapYear(year) == 1:
if mon > 2:
ydate += 1
#
#---- for the case the time format is seconds from 1998.1.1
#
elif mcf.chkNumeric(cdate):
out = Chandra.Time.DateTime(float(cdate)).date
atemp = re.split(':', out)
year = float(atemp[0])
ydate = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
else:
atemp = re.split(':', cdate)
year = float(atemp[0])
ydate = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
ydate = ydate + hh / 24.0 + mm / 1440.0 + ss / 86400.0
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fyear = year + ydate / base
return fyear
def convert_time(stime):
"""
convert the time format from <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> to fractional year
input: stime --- time in the fromat of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
output: fyear --- time in the foramt of the fractional year
"""
atemp = re.split('T', stime)
btemp = re.split('-', atemp[0])
ctemp = re.split(':', atemp[1])
year = float(btemp[0])
mon = int(float(btemp[1]))
day = float(btemp[2])
yday = day + m_list[mon-1]
hh = float(ctemp[0])
mm = float(ctemp[1])
ss = float(ctemp[2])
if tcnv.isLeapYear(btemp[0]):
base = 366.0
if mon > 2:
yday += 1
else:
base = 365.0
fyear = year + (yday + hh / 24.0 + mm / 1440.0 + ss / 86400.0) / base
return fyear
def set_start_dom(start_year, start_yday):
"""
finding starting dom date
input: start_year --- the year that the new dataset starts
start_yday --- the ydate that the new dataset starts
output: dom --- dom of the starting date
"""
dom = 0
don = 202 #---- this is not correct as real definition of dom but for now use this
for year in range(1999, (start_year + 1)):
if tcnv.isLeapYear(year) == 1:
yend = 367
else:
yend = 366
for yday in range(1, yend):
if year == 1999 and yday < 203:
continue
elif year == start_year and yday >= start_yday:
break
else:
dom += 1
return dom
def convert_time(otime):
"""
convert timer format from <year>:<ydate>:<hours>:<mins>:<sec> to fractional year
input: otime --- time in <year>:<ydate>:<hours>:<mins>:<sec>
output fyear --- time in fractional year
"""
#
#--- input data sometime comes with an extra ":" at the front; check whether it is the case
#
k = 0
if otime[0] == ':':
k = 1
atemp = re.split(':', otime)
year = float(atemp[k])
ydate = float(atemp[k+1])
hours = float(atemp[k+2])
mins = float(atemp[k+3])
secs = float(atemp[k+4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fday = hours / 24.0 + mins / 1440.0 + secs / 86400.0
fyear = year + (ydate + fday) / base
return fyear
def convert_time_format(stime):
atemp = re.split('T', stime)
btemp = re.split('-', atemp[0])
year = btemp[0]
mon = int(float(btemp[1]))
yday = int(float(btemp[2]))
if tcnv.isLeapYear(year) == 1:
alist = mday_list2
else:
alist = mday_list
if mon > 1:
for k in range(0, mon - 1):
yday += alist[k]
lyday = str(yday)
if yday < 10:
lyday = '00' + lyday
elif yday < 100:
lyday = '0' + lyday
date = year + ':' + lyday + ':' + atemp[1]
stday = Chandra.Time.DateTime(date).secs
return stday
def convert_to_fyear(val):
"""
convert sim temp database time format to fractional year
input: val --- sim temp time
output: fyear --- fractional year e.g. 2014.11345
"""
v = str(val)
year = v[0] + v[1] + v[2] + v[3]
year = float(year)
yday = v[4] + v[5] + v[6]
yday = float(yday)
hh = v[7] + v[8]
hh = float(hh)
mm = v[9] + v[10]
mm = float(mm)
ss = v[11] + v[12]
ss = float(ss)
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + (yday + hh / 24.0 + mm / 1440.0 + ss / 86400.0) / base
return fyear
def convert_time(otime):
"""
convert timer format from <year>:<ydate>:<hours>:<mins>:<sec> to fractional year
input: otime --- time in <year>:<ydate>:<hours>:<mins>:<sec>
output fyear --- time in fractional year
"""
#
#--- input data sometime comes with an extra ":" at the front; check whether it is the case
#
k = 0
if otime[0] == ':':
k = 1
atemp = re.split(':', otime)
year = float(atemp[k])
ydate = float(atemp[k + 1])
hours = float(atemp[k + 2])
mins = float(atemp[k + 3])
secs = float(atemp[k + 4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fday = hours / 24.0 + mins / 1440.0 + secs / 86400.0
fyear = year + (ydate + fday) / base
return fyear
def convert_time_to_year(tlist):
"""
converting Chandra time to fractional year
input: tlist --- a list of time in format of seconds from 1998.1.1
output: save --- a list of time in format of fractional year
"""
save = []
for ent in tlist:
ldate = Chandra.Time.DateTime(ent).date
atemp = re.split(':', ldate)
year = float(atemp[0])
yday = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fyear = year + (yday + hh / 24.0 + mm / 1440.0 + ss / 86400.0) / base
save.append(fyear)
return save
def print_month_list():
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
line = 'Sep99 41 71\n'
begin = 71
for iyear in range(1999, year+1):
if tcnv.isLeapYear(iyear) == 1:
month_list = month_list2
else:
month_list = month_list1
lyear = str(iyear)
syear = lyear[2] + lyear[3]
for month in range(1, 13):
if iyear == 1999 and month < 10:
continue
if iyear == year and month > mon:
break
lmon = tcnv.changeMonthFormat(month)
time = lmon + syear
daynum = month_list[month-1]
end = daynum + begin
line = line + time + ' ' + str(begin) + ' ' + str(end) + '\n'
begin = end
fo = open('/data/mta_www/mta_sim/Scripts/SIM_move/Outputs/months', 'w')
fo.write(line)
fo.close()
def change_to_fyear(stime):
"""
change time format from chadra time to fractional year
input: stime --- a list of time in seconds from 1998.1.1
output: ctime --- a list of time in fractional year
"""
ctime = []
for ent in stime:
ldate = Chandra.Time.DateTime(ent).date
atemp = re.split(':', ldate)
year = float(atemp[0])
yday = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fyear = year + (yday + hh / 24.0 + mm / 1440.0 + ss / 86400.0) / base
ctime.append(fyear)
return ctime
def collect_data(dir, col_list, outfile, eline):
#
#--- now loop through year and ydate
#
for year in range(2015, 2017):
for month in range(1, 13):
if year == 2014 and month < 12:
continue
chk = tcnv.isLeapYear(year)
if (year == 2016) and (month > 6):
break
print "Date year: " + str(year) + ' month: ' + str(
month) + ' : ' + str(dir)
try:
line = extract_new_data(dir, col_list, year, month)
except:
line = 'na'
cmd = 'rm ./mta*fits* ./Working_dir/mta*fits*'
os.system(cmd)
fo = open(outfile, 'a')
if line != 'na':
fo.write(line)
else:
dom = int(tcnv.YdateToDOM(year, yday))
line = str(dom) + '\t' + eline + '\n'
fo.write(line)
fo.close()
def convert_time_format(atime):
"""
convert time in seconds from 1998.1.1 to a fractional year
input: atime --- time in seconds from 1998.1.1
output: otime --- time in a fractional year
"""
date = Chandra.Time.DateTime(atime).date
atemp = re.split(':', date)
year = float(atemp[0])
yday = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
otime = year + yday / base + hh / 24.0 + mm / 1440.0 + ss / 86400.0
return otime
def convert_time_format(ltime):
"""
convert data fromat from <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss> to <yyyy>:<ddd>:<hh>:<mm>:<ss>
input: ltime time in the format of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
output: date time in the format of <yyyy>:<ddd>:<hh>:<mm>:<ss>
"""
atemp = re.split('T', ltime)
btemp = re.split('-', atemp[0])
year = int(float(btemp[0]))
mon = int(float(btemp[1]))
day = int(float(btemp[2]))
yday = mon_list[mon - 1] + day
if (tcnv.isLeapYear(year) == 1) and (mon > 2):
yday += 1
lyday = str(yday)
if yday < 10:
lyday = '00' + lyday
elif yday < 100:
lyday = '0' + lyday
date = str(year) + ':' + lyday + ':' + atemp[1]
return date
def change_ctime_to_ydate(cdate, yd=1):
"""
convert chandra time into fractional year or ydate
input: cdate --- chandra time
yd --- indicator to create fractional year (0) or ydate (1)
output: year --- the year of the date
date --- if yd==0, date in fractional year, otherwise, in ydate
"""
date = Chandra.Time.DateTime(cdate).date
atemp = re.split(':', date)
year = float(atemp[0])
date = float(atemp[1])
hh = float(atemp[2])
mm = float(atemp[3])
ss = float(atemp[4])
date += (hh + mm / 60.0 + ss / 3600.0) /24.0
if yd == 1:
return [year, date]
else:
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
date = year + date /base
return [year, date]
def find_time_span(year='', month='', mday=''):
"""
find time span for Thu to Thu nearest to a given date
input: year --- year. default: current year
month --- month. default: current month
mday --- month date. default: today's date
"""
#
#--- if date is given, fine week day and year date
#
if year != '':
input_time = str(year) + ':' + str(month) + ':' + str(mday)
tlist = time.strptime(input_time, "%Y:%m:%d")
wday = tlist.tm_wday
yday = tlist.tm_yday
#
#--- find today's date information (in local time)
#
else:
tlist = time.localtime()
year = tlist[0]
mon = tlist[1]
day = tlist[2]
wday = tlist[6]
yday = tlist[7]
#
#--- find the differnce to Wednesday (Thursday). wday starts on Monday (0)
#--- and set data collect date span
#
#diff = 5 - wday
diff = 3 - wday
if diff != 0:
yday += diff
if yday < 1:
year -= 1
yday = base - yday
syear = year
syday = yday - 8
#syday = yday - 7
if syday < 0:
syear -= 1
if tcnv.isLeapYear(syear) == 1:
base = 366
else:
base = 365
syday += base
#
#--- convert time into seconds from 1998.1.1
#
start = convertto1998sec(syear, syday)
stop = convertto1998sec(year, yday)
return [start, stop]
def find_time_span(year = '', month = '', mday = ''):
"""
find time span for Thu to Thu nearest to a given date
input: year --- year. default: current year
month --- month. default: current month
mday --- month date. default: today's date
"""
#
#--- if date is given, fine week day and year date
#
if year != '':
input_time = str(year) +':' + str(month) + ':' + str(mday)
tlist = time.strptime(input_time, "%Y:%m:%d")
wday = tlist.tm_wday
yday = tlist.tm_yday
#
#--- find today's date information (in local time)
#
else:
tlist = time.localtime()
year = tlist[0]
mon = tlist[1]
day = tlist[2]
wday = tlist[6]
yday = tlist[7]
#
#--- find the differnce to Wednesday (Thursday). wday starts on Monday (0)
#--- and set data collect date span
#
diff = 5 - wday
#diff = 3 - wday
if diff != 0:
yday += diff
if yday < 1:
year -= 1
yday = base - yday
syear = year
syday = yday - 8
#syday = yday - 7
if syday < 0:
syear -= 1
if tcnv.isLeapYear(syear) == 1:
base = 366
else:
base = 365
syday -= base
#
#--- convert time into seconds from 1998.1.1
#
start = convertto1998sec(syear, syday)
stop = convertto1998sec(year, yday)
return [start, stop]
def find_base_data(year):
if tcnv.isLeapYear(syear) == 1:
base = 366
else:
base = 365
return base
def find_date_and_year_for_report():
"""
find nearest Thursday date
input: none
output: date --- date of the nearest Thu in the format of mmdd (e.g. 0910)
year --- year of the nearest Thu
"""
#
#--- find today's date information (in local time)
#
tlist = time.localtime()
year = tlist[0]
mon = tlist[1]
day = tlist[2]
wday = tlist[6]
yday = tlist[7]
#
#--- find the differnce to Thursday. wday starts on Monday (0)
#
diff = 3 - wday
if diff != 0:
yday += diff
if yday < 1:
year -= 1
if tcnv.isLeapYear(year):
base = 366
else:
base = 365
yday = base - yday
#
#--- converting the year and ydate into the standard date output
#
tline = str(year) + ' ' +str(yday)
tlist = time.strptime(tline, "%Y %j")
year = tlist[0]
mon = tlist[1]
day = tlist[2]
#
#--- change the date foramt to mmdd (e.g. 0910)
#
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
date = smon + sday
year = str(year)
return [date, year]
def find_date_and_year_for_report():
"""
find nearest Thursday date
input: none
output: date --- date of the nearest Thu in the format of mmdd (e.g. 0910)
year --- year of the nearest Thu
"""
#
#--- find today's date information (in local time)
#
tlist = time.localtime()
year = tlist[0]
mon = tlist[1]
day = tlist[2]
wday = tlist[6]
yday = tlist[7]
#
#--- find the differnce to Thursday. wday starts on Monday (0)
#
diff = 3 - wday
if diff != 0:
yday += diff
if yday < 1:
year -= 1
if tcnv.isLeapYear(year):
base = 366
else:
base = 365
yday = base - yday
#
#--- converting the year and ydate into the standard date output
#
tline = str(year) + ' ' +str(yday)
tlist = time.strptime(tline, "%Y %j")
year = tlist[0]
mon = tlist[1]
day = tlist[2]
#
#--- change the date foramt to mmdd (e.g. 0910)
#
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
date = smon + sday
year = str(year)
return [date, year]
def update_acis_ctemp():
"""
"""
#
#--- read msid list
#
afile = house_keeping + 'msid_list_compaciscent'
data = ecf.read_file_data(afile)
acis_list = []
for ent in data:
atemp = re.split('\s+', ent)
acis_list.append(atemp[0])
for year in range(1999, 2019):
nyear = year
if tcnv.isLeapYear(year) == 1:
mon_list = mon_list2
else:
mon_list = mon_list1
for mon in range(0, 12):
if year == 1999:
if mon < 7:
continue
if year == 2018:
if mon > 1:
break
if mon == 11:
bday = mon_list[mon]
eday = 1
nyear += 1
else:
bday = mon_list[mon]
eday = mon_list[mon + 1]
cbday = str(bday)
if bday < 10:
cbday = '00' + cbday
elif bday < 100:
cbday = '0' + cbday
ceday = str(eday)
if eday < 10:
ceday = '00' + ceday
elif eday < 100:
ceday = '0' + ceday
start = str(year) + ':' + cbday + ':00:00:00'
stop = str(nyear) + ':' + ceday + ':00:00:00'
get_data(start, stop, year, acis_list)
def find_base(year):
"""
find how many days in a given year
input: year --- year in 4 digit
outpu: base --- either 365 or 366 (leap year)
"""
if tcnv.isLeapYear(year):
base = 366
else:
base = 365
return base
def fyear_to_dom(dlist):
dom_list = []
for ent in dlist:
year = int(ent)
fy = ent - year
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
ydate = int(base * fy)
dom = tcnv.YdateToDOM(year, ydate)
dom_list.append(dom)
return dom_list
def adjust_year_date(byear, x):
if tcnv.isLeapYear(byear) == 1:
base = 366
else:
base = 365
nx = []
for ent in x:
if ent[0] != byear:
val = float(ent[1]) + base
nx.append(val)
else:
nx.append(float(ent[1]))
return nx
def compute_dom(eyear, eyday):
if eyear == 1999:
dom = eyday - 201
else:
dom = 164
for year in range(2000, eyear):
if tcnv.isLeapYear(year) == 1:
dom += 366
else:
dom += 365
dom += eyday
return dom
def change_time_format_fyear(date):
atemp = re.split(':', date)
year = int(atemp[0])
yday = int(atemp[1])
hh = int(atemp[2])
mm = int(atemp[3])
ss = int(atemp[4])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fyear = year + (yday + hh / 24.0 + mm / 1440.0 + ss / 886400.0) / base
return fyear
def convert_date_format(date, hhmmss=''):
"""
convert date format from <Mmm> <dd> <yyyy> to seconds from 1998.1.1
input: date --- date in the fromat of <Mmm> <dd> <yyyy>
hhmmss --- <hh>:<mm>:<ss>; default: '' which creates 00:00:00
output: date --- date in seconds from 1998.1.1
"""
if date.lower() == 'na':
return 'na'
else:
atemp = re.split('\s+', date)
for k in range(0, len(cmon)):
if (cmon[k] == atemp[0]) or (cmon[k] == atemp[1]):
pos = k
break
try:
day = int(float(atemp[1]))
except:
val = atemp[0].strip()
val = val.replace(',', '')
day = int(float(val))
year = int(float(atemp[2]))
if tcnv.isLeapYear(year) == 1:
yday = dmon2[k] + day
else:
yday = dmon1[k] + day
cday = str(yday)
if yday < 10:
cday = '00' + cday
elif yday < 100:
cday = '0' + cday
if hhmmss == '':
tdate = str(year) + ':' + cday + ':00:00:00'
else:
mc = re.search(':', hhmmss)
if mc is not None:
tdate = str(year) + ':' + cday + ':' + hhmmss
else:
tdate = str(year) + ':' + cday + ':00:00:00'
tdate = Chandra.Time.DateTime(tdate).secs
return tdate
def create_monthly_bins(ystart, ystop, mstop):
"""
create a month wide bin for given periods
input: ystart --- starting year
ystop --- stopping year
mstop --- stopping month of the stopping month
output: [blist, elist] a list of lists of starting and stoping period in fractional year
"""
interval1 = [
0.0, 31.0, 59.0, 90.0, 120.0, 151.0, 181.0, 212.0, 243.0, 273.0, 304.0,
334.0, 365.0
]
interval2 = [
0.0, 31.0, 60.0, 91.0, 121.0, 152.0, 182.0, 213.0, 244.0, 274.0, 305.0,
335.0, 366.0
]
blist = []
elist = []
for year in range(ystart, ystop + 1):
#
#--- check leap year
#
if tcnv.isLeapYear(year) == 1:
interval = interval2
base = 366.0
else:
interval = interval1
base = 365.0
#
#--- go around 12 months
#
for i in range(0, 12):
if year == ystop and i >= mstop:
break
begin = year + interval[i] / base
end = year + interval[i + 1] / base
if int(end) > year:
end = year + 1
blist.append(begin)
elist.append(end)
return [blist, elist]
def find_starting_date():
"""
set starting and stopping time from the last entry of a fits file
input: none but read from hvpsstat_data.fits
output: start --- starting time <yyyy>-<mm>-<dd>T00:00:00
stop --- stoping time <yyyy>-<mm>-<dd>T23:59:59
"""
test = dpath + 'hvpsstat_data.fits'
ltime = ecf.find_the_last_entry_time(test)
#
#--- convert time from sec from 1998.1.1 to year and ydate
#
out = Chandra.Time.DateTime(ltime).date
atemp = re.split(':', out)
year = int(float(atemp[0]))
yday = int(float(atemp[1]))
yday += 1
#
#--- check whether the date crosses the year end
#
if tcnv.isLeapYear(year) == 1:
if yday > 366:
yday = 1
year += 1
else:
if yday > 365:
yday = 1
year += 1
#
#--- convert to month and mday
#
[mon, day] = tcnv.changeYdateToMonDate(year, yday)
lmon = str(mon)
if mon < 10:
lmon = '0' + lmon
lday = str(day)
if day < 10:
lday = '0' + lday
start = str(year) + '-' + lmon + '-' + lday + 'T00:00:00'
stop = str(year) + '-' + lmon + '-' + lday + 'T23:59:59'
return [start, stop]
def convert_date_format(sdate):
"""
convert time format from <Mmm> <dd> <yyy> <hh>:<mm><AM/PM> to <yyyy>:<ddd>:<hh>:<mm>:<ss>
input sdate --- date in <Mmm> <dd> <yyy> <hh>:<mm><AM/PM>
output ltime --- date in <yyyy>:<ddd>:<hh>:<mm>:<ss>
"""
atemp = re.split('\s+', str(sdate))
mon = atemp[0]
for k in range(0, 12):
if mon.upper() == m_list[k]:
mpos = k
break
date = int(float(atemp[1]))
year = int(float(atemp[2]))
if tcnv.isLeapYear(year) == 1:
m_start = m_start2
else:
m_start = m_start1
yday = m_start[mpos] + date
syday = str(yday)
if yday < 10:
syday = '00' + syday
elif yday < 100:
syday = '0' + syday
tpart = atemp[3]
mc = re.search('AM', tpart)
if mc is not None:
add = 0
else:
add = 12
tpart = tpart[:-2]
btemp = re.split(':', tpart)
hh = int(float(btemp[0])) + add
shh = str(hh)
if hh < 10:
shh = '0' + shh
ltime = str(year) + ':' + str(syday) + ':' + str(shh) + ':' + str(
btemp[1]) + ':00'
return ltime
def convert_to_ydate_list(x):
nx = []
byear = ''
for val in x:
cdata = change_ctime_to_ydate(val)
if byear == '':
byear = cdata[0]
if tcnv.isLeapYear(byear) == 1:
base = 366
else:
base = 365
if cdata[0] > byear:
nx.append(cdata[1] + base)
else:
nx.append(cdata[1])
return nx
def read_data(file):
"""
read data from a given file
Input: file --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
date_list = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0])== False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
dom = float(atemp[10])
(year, ydate) = tcnv.DOMtoYdate(dom)
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + ydate/base
date_list.append(fyear)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede*ede* ((ferr*ferr) / (fwhm*fwhm)))
error_list.append(error)
return [date_list, ede_list, error_list]
def update_sim_offset():
"""
"""
for year in range(1999, 2019):
nyear = year
if tcnv.isLeapYear(year) == 1:
mon_list = mon_list2
else:
mon_list = mon_list1
for mon in range(0, 12):
if year == 1999:
if mon < 7:
continue
if year == 2018:
if mon > 1:
break
if mon == 11:
bday = mon_list[mon]
eday = 1
nyear += 1
else:
bday = mon_list[mon]
eday = mon_list[mon + 1]
cbday = str(bday)
if bday < 10:
cbday = '00' + cbday
elif bday < 100:
cbday = '0' + cbday
ceday = str(eday)
if eday < 10:
ceday = '00' + ceday
elif eday < 100:
ceday = '0' + ceday
start = str(year) + ':' + cbday + ':00:00:00'
stop = str(nyear) + ':' + ceday + ':00:00:00'
get_data(start, stop, year)
def set_x_range(xdata):
"""
set several x axis related values
input: xdata --- x data
output: ndata --- x data set for the short time span (less than 2 yrs)
xmin --- x min
xmax --- x max
xlabel --- label of x axis
xtext --- x position of the text to be printed
"""
xmin = min(xdata)
year = int(xmin)
xmax = max(xdata)
xdiff = xmax - xmin
if xdiff < 2.0:
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
ndata = []
for ent in xdata:
val = base * (ent - year)
ndata.append(val)
xlabel = 'Time (year=' + str(year) + ' yday)'
xmin = 0
xmax = base
else:
ndata = xdata
xmin = int(xmin)
xmax = int(xmax) + 1
xlabel = 'Time (year)'
xdiff = xmax - xmin
xtext = xmin + 0.02 * xdiff
return [ndata, xmin, xmax, xlabel, xtext]
def current_time():
"""
return current time in fractional year
input: none
output: fyear
"""
otime = time.gmtime()
year = otime.tm_year
yday = otime.tm_yday
hr = otime.tm_hour
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + (yday + hr / 24.0) / base
return fyear
def current_time():
"""
return current time in fractional year
input: none
output: fyear
"""
otime = time.gmtime()
year = otime.tm_year
yday = otime.tm_yday
hr = otime.tm_hour
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + (yday + hr/24.0) / base
return fyear
def read_data(file):
"""
read data from a given file
Input: file --- input file name
Output: date_list --- a list of date
ede_list --- a list of ede value
error_list --- a list of computed ede error
"""
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
date_list = []
ede_list = []
error_list = []
for ent in data:
atemp = re.split('\s+', ent)
if mcf.chkNumeric(atemp[0]) == False:
continue
fwhm = float(atemp[2])
ferr = float(atemp[3])
ede = float(atemp[4])
dom = float(atemp[10])
(year, ydate) = tcnv.DOMtoYdate(dom)
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + ydate / base
date_list.append(fyear)
ede_list.append(ede)
#
#--- the error of EdE is computed using FWHM and its error value
#
error = math.sqrt(ede * ede * ((ferr * ferr) / (fwhm * fwhm)))
error_list.append(error)
return [date_list, ede_list, error_list]
def set_x_range(xdata):
"""
setting x range. if less than 2 years, use yday, otherwise, fractional year
input: xdata --- x data
output: ndata --- modified data (for the case if it is in yday)
xmin --- x min
xmax --- x max
xlabel --- label for x axis
xtext --- x location for the text display
"""
xmin = min(xdata)
year = int(xmin)
xmax = max(xdata)
xdiff = xmax - xmin
if xdiff < 2.0:
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
ndata = []
for ent in xdata:
val = base * (ent - year)
ndata.append(val)
xlabel = 'Time (yday @ year=' + str(year) + ')'
xmin = 0
xmax = base
else:
ndata = xdata
xmin = int(xmin)
xmax = int(xmax) + 1
xlabel = 'Time (year)'
xdiff = xmax - xmin
xtext = xmin + 0.1 * xdiff
return [ndata, xmin, xmax, xlabel, xtext]
def create_monthly_bins(ystart, ystop, mstop):
"""
create a month wide bin for given periods
input: ystart --- starting year
ystop --- stopping year
mstop --- stopping month of the stopping month
output: [blist, elist] a list of lists of starting and stoping period in fractional year
"""
interval1 = [0.0, 31.0, 59.0, 90.0, 120.0, 151.0, 181.0, 212.0, 243.0, 273.0, 304.0, 334.0, 365.0]
interval2 = [0.0, 31.0, 60.0, 91.0, 121.0, 152.0, 182.0, 213.0, 244.0, 274.0, 305.0, 335.0, 366.0]
blist = []
elist = []
for year in range(ystart, ystop+1):
#
#--- check leap year
#
if tcnv.isLeapYear(year) == 1:
interval = interval2
base = 366.0
else:
interval = interval1
base = 365.0
#
#--- go around 12 months
#
for i in range(0, 12):
if year == ystop and i >= mstop:
break
begin = year + interval[i] / base
end = year + interval[i+1] / base
if int(end) > year:
end = year + 1
blist.append(begin)
elist.append(end)
return [blist, elist]
def stime_to_frac_year(stime):
"""
convert seconds from 1998.1.1 to fractional year format
input: stime --- seconds from 1998.1.1
etime --- time in fractinal year;, e.g., 2012.223
"""
etime = Ebase_t + stime
etime = time.localtime(etime)
etime = time.strftime( '%Y-%j', etime)
atemp = re.split('-', etime)
year = float(atemp[0])
ydate = float(atemp[1])
if tcnv.isLeapYear(year):
base = 366
else:
base = 365
etime = year + ydate / base
return etime
def convert_time(otime):
"""
convert time format from <year><ydate>.<hh><mm><ss> to frac year
input: otime --- time in e.g. 2014059.122333.1
output: fyear --- fractional year, e.g. 2014.1630585
"""
year = float(otime[0:4])
ydate = float(otime[4:7])
hours = float(otime[8:10])
mins = float(otime[10:12])
secs = float(otime[12:14])
if tcnv.isLeapYear(year) == 1:
base = 366.0
else:
base = 365.0
fday = hours / 24.0 + mins / 1440.0 + secs / 86400.0
fyear = year + (ydate + fday) / base
return fyear
def check_date(comp_test=''):
"""
check wether there is an output directory and if it is not, create one
Input: comp_test --- if it is "test", the test data is used
Output: uyear --- the current year
umon --- the current month
mon_name --- the current output direcotry (if it is not there, created)
"""
start_year = []
start_month = []
start_date = []
end_year = []
end_month = []
end_date = []
tot_ent = 1
if comp_test == 'test':
#
#--- test case, date is fixed
#
tyear = 2013
tmon = 2
tday = 13
uyear = tyear
umon = tmon
else:
#
#--- find today's date
#
[uyear, umon, uday, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
tyear = uyear
tmon = umon
tday = uday
end_year.append(tyear)
end_month.append(tmon)
end_date.append(tday)
#
#--- check 10 days ago
#
lday = tday - 10
lmon = tmon
lyear = tyear
if lday < 1:
#
#--- if 10 days ago is the last month, set starting time in the last month
#
tot_ent = 2
start_year.append(tyear)
start_month.append(tmon)
start_date.append(1)
if tmon == 5 or tmon == 7 or tmon == 10 or tmon == 12:
lday += 30
lmon = tmon - 1
end_year.append(tyear)
end_month.append(lmon)
end_date.append(30)
start_year.append(tyear)
start_month.append(lmon)
start_date.append(lday)
elif tmon == 2 or tmon == 4 or tmon == 6 or tmon == 8 or tmon == 9 or tmon == 11:
lday += 31
lmon = tmon - 1
end_year.append(tyear)
end_month.append(lmon)
end_date.append(31)
start_year.append(tyear)
start_month.append(lmon)
start_date.append(lday)
elif tmon == 3:
#
#--- last month is in Feb
#
fday = 28
if tcnv.isLeapYear(tyear) > 0:
fday = 29
lday += fday
lmon = tmon -1
end_year.append(tyear)
end_month.append(lmon)
end_date.append(fday)
start_year.append(tyear)
start_month.append(lmon)
start_date.append(lday)
elif tmon == 1:
#
#--- last month is the year before
#
lday += 31
lmon = 12
lyear = tyear -1
end_year.append(tyear)
end_month.append(lmon)
end_date.append(31)
start_year.append(tyear)
start_month.append(lmon)
start_date.append(lday)
else:
#
#--- 10 days ago is in the same month
#
start_year.append(lyear)
start_month.append(lmon)
start_date.append(lday)
#
#--- reverse the list
#
start_year.reverse()
start_month.reverse()
start_date.reverse()
end_year.reverse()
end_month.reverse()
end_date.reverse()
#
#--- start checking whether directory exists. if not create it
#
no = 0
for dmon in(start_month):
cmonth = tcnv.changeMonthFormat(dmon) #--- convert digit to letter month
ucmon = cmonth.upper()
mon_name = web_dir + '/' + ucmon + str(start_year[no])
no += 1
chk = mcf.chkFile(mon_name)
if chk == 0:
cmd = 'mkdir ' + mon_name
os.system(cmd)
return (uyear, umon, mon_name)
def set_data_period(year, sdate, edate):
"""
create a list of dates to be examined
input: year --- year of the date
sdate --- starting yday
edate --- ending ydate
these three can be <blank>. if that is the case, it will fill from
the date of the last data entry to today's date
output: dperiod --- a list of dates in the formant of [[2015, 199], [2015, 200], ...]
"""
if year != '':
dperiod = []
for yday in range(sdate, edate+1):
dperiod.append([year, yday])
else:
#
#--- find today's date
#
today = time.localtime()
year = today.tm_year
yday = today.tm_yday
#
#--- find the last date of the data entry
#--- entry format: 2015365.21252170 16.4531 27.0 33.0 10 174040 0 0 28.4
#
file = data_dir + 'tsc_temps.txt'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
lent = data[-1]
atemp = re.split('\s+', lent)
btemp = re.split('\.', atemp[0])
ldate = btemp[0]
dyear = ldate[0] + ldate[1] + ldate[2] + ldate[3]
dyear = int(float(dyear))
dyday = ldate[4] + ldate[5] + ldate[6]
dyday = int(float(dyday))
#
#--- check whether it is a leap year
#
lchk = tcnv.isLeapYear(dyear)
if lchk == 1:
base = 366
else:
base = 365
#
#--- now start filling the data period (a pair of [year, ydate])
#
dperiod = []
#
#--- for the case, year change occurred
#
if dyear < year:
for ent in range(dyday, base+1):
dperiod.append([dyear, ent])
for ent in range(1, yday+1):
dperiod.append([year, ent])
#
#--- the period in the same year
#
else:
for ent in range(dyday, yday+1):
dperiod.append([year, ent])
#
#--- return the result
#
return dperiod
def update_weekly_telem(year, byear, mon):
"""
adjusting weekly_telem.pro
--- line 55 is the check of monthly boundary. it needs to be updated.
input: year --- year in 4 digit
bear --- year of beginning date
mon --- month in
output: <wdir>/Telem/weekly_telem.pro updated
"""
#
#--- convert year, byear, and mon to integer
#
year = int(float(year))
byear = int(float(byear))
mon = int(float(mon))
#
#--- if the year changes from the beginning of the period to the end of the period
#--- use the year of the beginning of the period and change the monthe Dec (12)
#
if byear < year:
year = byear
mon = 12
else:
mon -= 1 #--- a quick fix for the month change mon was "this month"
#--- and the transition occurs from the last month to this month
mon_plus = (0, 32, 29, 32, 31, 32, 31, 32, 32, 31, 32, 31, 32)
nyear = year
nmon = mon + 1
if nmon > 12:
nyear += 1
nmon = 1
lday = mon_plus[mon]
#
#--- if it is a leap year, the end of Feb is 29
#
if mon == 2:
if tcnv.isLeapYear(year):
lday += 1
smon = str(mon)
if mon < 10:
smon = '0' + smon
snmon = str(nmon)
if nmon < 10:
snmon = '0' + snmon
this = str(year) + smon + str(lday)
that = str(nyear) + snmon + '01'
file = tdir + 'weekly_telem_template'
f = open(file, 'r')
data = f.read()
f.close()
data = data.replace('#THIS#', this)
data = data.replace('#THAT#', that)
out = wdir + 'Telem/weekly_telem.pro'
fo = open(out, 'w')
fo.write(data)
fo.close()
cmd = 'chmod 755 ' + out
os.system(cmd)
def print_run_rad():
"""
create today's run_rad.pro idl script
"""
#
#--- find today's date
#
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#---if this is the first of the month, compute the last month
#
if day < 2:
subtract = 1
else:
subtract = 0
cyear = year
lmon = mon - subtract
if lmon < 1:
lmon = 12
cyear = year -1
#
#--- choose a correct month list depending on whether this is the leap year
#
if tcnv.isLeapYear(cyear) == 1:
mon_list = mon_list1
else:
mon_list = mon_list2
if lmon == 1:
start_day = '001'
else:
sday = float(mon_list[lmon-2]) + 1
start_day = str(sday)
if sday < 10:
start_day = '00' + start_day
elif sday < 100:
start_day = '0' + start_day
last_day = mon_list[lmon-1]
#
#--- convert the month from a numeric to letter
#
smon = tcnv.changeMonthFormat(lmon)
smon = smon.lower()
#
#--- set date format to the appropriate ones
#
today = str(yday)
if yday < 100:
today = '0' + today
elif today < 10:
today = '00' + today
day30 = yday - 30
if day30 < 0:
day30 = 366 + day30
tday_year = str(year -1)
else:
tday_year = str(year)
if day30 < 10:
day30 = '00' + str(day30)
elif day30 < 100:
day30 = '0' + str(day30)
else:
day30 = str(day30)
lmon_year = str(cyear)
syear = lmon_year[2] + lmon_year[3]
last_year = str(year -1)
syear2 = last_year[2] + last_year[3]
monyear = smon + syear
#
#--- now print out the run_rad.pro
#
line = "x = mta_rad('" + tday_year + ":" + str(day30) +"')\n"
line = line + "spawn, 'mv rad_cnts.gif rad_cnts_curr.gif'\n"
line = line + "spawn, 'mv rad_use.gif rad_use_curr.gif'\n"
line = line + "print, 'Done current'\n"
line = line + "retall\n"
line = line + "x = mta_rad('" + lmon_year + ":" + start_day + "','" + lmon_year + ":" + last_day + "')\n"
line = line + "spawn, 'mv rad_cnts.gif rad_cnts_" + monyear + ".gif'\n"
line = line + "spawn, 'mv rad_use.gif rad_use_" + monyear + ".gif'\n"
line = line + "spawn, 'mv eph_diff.gif eph_diff_" + monyear + ".gif'\n"
line = line + "spawn, 'mv mon_diff.gif mon_diff_" + monyear + ".gif'\n"
line = line + "spawn, 'mv per_diff.gif per_diff_" + monyear + ".gif'\n"
line = line + "spawn, 'mv xper_diff.gif mon_per_diff_" + monyear + ".gif'\n"
line = line + "print, 'Done " + monyear + "'\n"
line = line + "x = mta_rad('" + last_year + ":001','" + lmon_year + ":001')\n"
line = line + "spawn, 'mv rad_cnts.gif rad_cnts_" + syear2 + ".gif'\n"
line = line + "spawn, 'mv rad_use.gif rad_use_" + syear2 + ".gif'\n"
line = line + "spawn, 'mv eph_diff.gif eph_diff_" + syear2 + ".gif'\n"
line = line + "spawn, 'mv mon_diff.gif mon_diff_" + syear2 + ".gif'\n"
line = line + "spawn, 'mv per_diff.gif per_diff_" + syear2 + ".gif'\n"
line = line + "spawn, 'mv xper_diff.gif mon_per_diff_" + syear2 + ".gif'\n"
line = line + "print, 'Done " + syear2 + "'\n"
line = line + "x = mta_rad('" + last_year + ":" + today + "','" + lmon_year + ":" + today + "')\n"
line = line + "spawn, 'mv rad_cnts.gif rad_cnts_last_one_year.gif'\n"
line = line + "spawn, 'mv rad_use.gif rad_use_last_one_year.gif'\n"
line = line + "spawn, 'mv eph_diff.gif eph_diff_last_one_year.gif'\n"
line = line + "spawn, 'mv mon_diff.gif mon_diff_last_one_year.gif'\n"
line = line + "spawn, 'mv per_diff.gif per_diff_last_one_year.gif'\n"
line = line + "spawn, 'mv xper_diff.gif mon_per_diff_last_one_year.gif'\n"
line = line + "print, 'Done Last One Year\n"
line = line + "x = mta_rad()\n"
line = line + "spawn, 'mv rad_cnts.gif rad_cnts_all.gif'\n"
line = line + "spawn, 'mv rad_use.gif rad_use_all.gif'\n"
line = line + "spawn, 'mv eph_diff.gif eph_diff_all.gif'\n"
line = line + "spawn, 'mv mon_diff.gif mon_diff_all.gif'\n"
line = line + "spawn, 'mv per_diff.gif per_diff_all.gif'\n"
line = line + "print, 'Done all'\n"
line = line + "retall\n"
line = line + "exit\n"
# fo = open('test.pro', 'w')
fo = open('run_rad.pro', 'w')
fo.write(line)
fo.close()
def create_monthly_focal_temp(year, month, eyear, emonth):
"""
main script to create a monthly focal temp trend plots
input: year --- start year in the format of yyyy (e.g. 2015)
month --- start month in the format of mm (e.g. 1 or 12)
eyear --- stopping year
emonth --- stopping month
output: a direcotry containing templete (e.g. Sep10)
"""
oned = 86400
syear = str(year) #--- 4 digit year
yrd2 = syear[2] + syear[3] #--- 2 digit year
year = int(float(year)) #--- integer year
smon = str(month)
mon = int(float(smon)) #--- integer month
lmon = tcnv.changeMonthFormat(mon) #--- month in letter (e.g.Mar)
sday = '01' #--- two digit mday
day = 1 #--- integer mday
start = tcnv.convertDateToCTime(year, mon, day, 0, 0, 0)
seyear = str(eyear) #--- 4 digit year
eyrd2 = seyear[2] + seyear[3] #--- 2 digit year
eyear = int(float(eyear)) #--- integer year
semon = str(emonth)
emon = int(float(semon)) #--- integer month
lemon = tcnv.changeMonthFormat(emon) #--- month in letter (e.g.Mar)
seday = '01' #--- two digit mday
eday = 1 #--- integer mday
stop = tcnv.convertDateToCTime(eyear, emon, eday, 0, 0, 0)
#
#--- set plot tick interval
#
if mon == 2:
dlist = [10, 20, 28]
if tcnv.isLeapYear(year) == 1:
dlist = [ 10, 20, 29]
else:
dlist = [10, 20, 30]
sdlist = [start]
for ent in dlist:
stime = tcnv.convertDateToCTime(year, mon, ent, 0, 0, 0)
sdlist.append(stime)
#
#--- focal temp file name
#
fptemp = 'erad_' + lmon.lower() + yrd2 + '.gif'
fpext_range = str(start)+' '+ str(stop)
fpstart = str(start)
fplsub = '"' + lmon.lower() + '01","' + lmon.lower() + '10","' + lmon.lower() + '20","' + lmon.lower() + str(dlist[2]) + '"'
fpdsub = str(sdlist[0]) + ', ' + str(sdlist[1]) + ', ' + str(sdlist[2]) + ', ' + str(sdlist[3])
#
#--- copy command scripts and a couple of others needed
#
cmd = 'cp -f ' + tdir + 'get_ftemp_data.perl ' + wdir + '.'
os.system(cmd)
cmd = 'cp -f ' + tdir + 'test ' + wdir + '.'
os.system(cmd)
cmd = 'cp -f ' + tdir + 'run_temp ' + wdir + '.'
os.system(cmd)
cmd = 'rm -rf param'
os.system(cmd)
cmd = 'mkdir param'
os.system(cmd)
#
#--- create idl script
#
tfile = tdir + 'plot_erad_time_month.pro'
f = open(tfile, 'r')
input = f.read()
f.close()
input = input.replace('#START#', str(start))
input = input.replace('#SDATELIST#', fpdsub)
input = input.replace('#LDATELIST#', fplsub)
input = input.replace('#GIFNAME#', fptemp)
ofile = wdir + 'plot_erad_time_month.pro'
fo = open(ofile, 'w')
fo.write(input)
fo.close()
#
#--- run the scripts
#
run_focal_temp_data(start, stop)
cmd = 'mv ' + fptemp + ' ./Plots/.'
os.system(cmd)
#
#--- clean up
#
cmd = 'rm -rf param get_ftemp_data.perl test run_temp plot_erad_time_month.pro out'
os.system(cmd)
def print_html(year, mon):
"""
create and/or update radiation related html page
"""
#
#--- find today's date
#
if year == '':
[year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#--- find out the last month
#
# cyear = year
# lmon = mon -1
# if lmon < 1:
# lmon = 12
# cyear = year -1
#
#--- for the case year and mon are given
#
cyear = year
lmon = mon
#
#--- choose a correct month list depending on whether this is the leap year
#
if tcnv.isLeapYear(cyear) == 1:
mon_list = mon_list1
else:
mon_list = mon_list2
last_day = mon_list[lmon-1]
#
#--- convert the month from a numeric to letter
#
umon = tcnv.changeMonthFormat(lmon)
smon = umon.lower()
lmon_year = str(cyear)
syear = lmon_year[2] + lmon_year[3]
last_year = str(year -1)
syear2 = last_year[2] + last_year[3]
monyear = smon + syear
#
#--- set output html page names
#
year_html = 'all' + syear + '.html'
mon_html = monyear + '.html'
rad_html = 'rad_time_' + monyear + '.html'
#
#--- read yearly html page template
#
data = open('./Template/yearly_template', 'r').read()
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#SYEAR#$', syear)
fo = open(year_html, 'w')
fo.write(data)
fo.close()
#
#--- read monthly html page template
#
data = open('./Template/monthly_template', 'r').read()
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#UMON#$',umon )
data = data.replace('$#MONYEAR#$', monyear)
fo = open(mon_html, 'w')
fo.write(data)
fo.close()
#
#--- read rad_time html page template
#
data = open('./Template/rad_time_template', 'r').read()
data = data.replace('$#LMONTH#$', fmon_list[mon-2])
data = data.replace('$#FYEAR#$', str(year))
data = data.replace('$#MONYEAR#$', monyear)
fo = open(rad_html, 'w')
fo.write(data)
fo.close()
def run_script():
for year in range(1999,2015):
leap = tcnv.isLeapYear(year)
if leap == 1:
dend = 367
else:
dend = 366
syear = str(year)
lyear = syear[2] + syear[3]
for yday in range(1,dend):
if year == 1999 and yday < 239:
continue
if year == 2014 and yday > 316:
break
print "Year: " + str(year) + ' Ydate: ' + str(yday)
[month, date] =tcnv.changeYdateToMonDate(year, yday)
smonth = str(month)
if month < 10:
smonth = '0' + smonth
sdate = str(date)
if date < 10:
sdate = '0' + sdate
start = smonth + '/' + sdate + '/' + lyear + ',00:00:00'
stop = smonth + '/' + sdate + '/' + lyear + ',23:59:59'
line = 'operation = retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = telem\n'
line = line + 'level = raw\n'
line = line + 'tstart = ' + start + '\n'
line = line + 'tstop = ' + stop + '\n'
line = line + 'go\n'
fo = open(zspace, 'w')
fo.write(line)
fo.close()
cmd1 = "/usr/bin/env PERL5LIB="
cmd2 = ' echo ' + hakama + ' |arc4gl -U' + dare + ' -Sarcocc -i' + zspace
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv0)
mcf.rm_file(zspace)
cmd = 'ls * > ' + zspace
os.system(cmd)
test = open(zspace, 'r').read()
mc = re.search('sto', test)
if mc is not None:
os.system('rm *log*')
os.system('gzip -fd *gz')
os.system('ls *.sto > xtmpnew')
os.system('nice ./filters_ccdm')
esd.extract_sim_data()
os.system('rm -rf *.sto *.tl')
def create_monthly(year='', mon=''):
"""
create monthly report
input: year --- year; if it is "", the year of the last month will be used
mon --- month;if it is "", the month value of the last month is used
output: monthly report in /data/mta4/www/REPORTS/MONTLHY/<yyyy><MMM>/
"""
#
#--- if year and month are not given, use the last month's month and year value
#
if year == '':
#
#--- find today's date
#
ltime = tcnv.currentTime()
year = ltime[0]
lyear = str(year) #--- '2016'
#
#--- set the last month's month and year
#
mon = ltime[1] -1
if mon < 1:
mon = 12
year -= 1
lyear = str(year) #--- '2016'
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon #--- e.g. 03 or 11
lmon = tcnv.changeMonthFormat(mon) #--- e.g. Mar or Nov
lmonyr = lmon.lower() + lyear[2] + lyear[3] #--- e.g. jan16
lm_y = cmon + '_' + lyear #--- e.g. 03_2016
#
#--- set output directory
#
odir = '/data/mta4/www/REPORTS/MONTHLY/' + str(year) + lmon.upper()
cmd = 'mkdir ' + odir
os.system(cmd)
odir = odir + '/'
#
#--- set month interval depending on leap year or not
#
if tcnv.isLeapYear(year) == 0:
sdate = ['001', '032', '060', '091', '121', '152', '182', '213', '244', '274', '305', '335']
edate = ['032', '060', '091', '121', '152', '182', '213', '244', '274', '305', '335', '001']
else:
sdate = ['001', '032', '061', '092', '122', '153', '183', '214', '245', '275', '306', '336']
edate = ['032', '061', '092', '122', '153', '183', '214', '245', '275', '306', '336', '001']
#
#--- set start and stop time of the month in <yyyy>:<ddd>:00:00:00 format
#
tstart = str(year) + ':' + sdate[mon-1] + ':00:00:00'
eyear = year
if mon == 12:
eyear = year + 1
tstop = str(eyear) + ':' + edate[mon-1] + ':00:00:00'
#
#--- create configulation plot
#
cmd = "cd /data/mta/Script/Month/Config/; create_config_plot.py " + tstart + ' ' + tstop
os.system(cmd)
#
#--- create CTI plots
#
os.system("cd /data/mta/Script/Month/CTI/; monthly_report_cti_avg_plots.py")
os.system("cd /data/mta/Script/Month/CTI/; monthly_report_cti_avg_plots_two_section.py")
#
#--- create Focal Plane temperature plots
#
os.system("cd /data/mta/Script/Month/FOCAL/; run_all_focal_scripts")
#
#--- create ACIS SIB plots
#
os.system("cd /data/mta/Script/Month/SIB/; sib_monthly_report_plot.py")
#
#--- create SIM movement plots
#
os.system("cd /data/mta/Script/Month/SIM/; create_monthly_sim_plots")
#
#--- copy the created plots to the report directory
#
cmd = "cp /data/mta/Script/Month/Config/rad_use_*.png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/CTI/Plots/*png " + odir
os.system(cmd)
cmd = "cp -rf /data/mta/Script/Month/CTI/Data " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/FOCAL/Plots/*png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/FOCAL/Plots/*gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_ccd_plot_front_side.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/SIB/*png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/Focus/foc_acis_hetg.gif " + odir
os.system(cmd)
cmd = "cp /data/mta/Script/Month/SIM/*.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_rad/mon_per_diff_last_one_year.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_rad/rad_cnts_" + lmonyr + ".gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/IRU/Plots/" + lmonyr + "_1h_bias.gif " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
os.system(cmd)
#
#--- move the plots to past plot saving directories
#
os.system("mv -f /data/mta/Script/Month/Config/*.png /data/mta/Script/Month/Config/Plots/.")
os.system("mv -f /data/mta/Script/Month/FOCAL/Plots/*gif /data/mta/Script/Month/FOCAL/Plots/Past/.")
os.system("mv -f /data/mta/Script/Month/SIB/*png /data/mta/Script/Month/SIB/Plots/.")
#
#--- copy other plot files we need for the report
#
cmd = "cp /data/mta4/www/DAILY/mta_pcad/IRU/Plots/" + lmonyr + "_1h_bias.gif " + odir
os.system(cmd)
cmd = 'cp /data/mta4/www/DAILY/mta_rad/rad_cnts_' + lmonyr + ".gif " + odir
os.system(cmd)
#
#--- sun spot cycle download
#
cmd = 'wget -q -O'+ odir + '/solar-cycle-sunspot-number.gif http://services.swpc.noaa.gov/images/solar-cycle-sunspot-number.gif'
os.system(cmd)
#
#--- now copy month depend plot files and set the template file for the month
#
if mon == 1 or mon == 7:
cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/I3cp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/S3cp.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Overclock/ccd2.png " + odir + 'ccd2_oc.png'
os.system(cmd)
cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Sub/ccd2.png " + odir + 'ccd2_sub.png'
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_2.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_6.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_7.png " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_11.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY1.html' #--- template file name
elif mon == 2 or mon == 8:
cmd = "cp /data/mta4/www/DAILY/mta_src/ACIS-I_d_tyear10_psf.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_xy.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_rnd.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rnd.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_ravg.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_ravg.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_snr.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_snr.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_rot.gif " + odir
os.system(cmd)
cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rot.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/acis_point_err.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_i_point_err.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_s_point_err.gif " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY2.html'
elif mon == 3 or mon == 9:
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_cti.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_obs.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_cti.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_obs.gif " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd5.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY3.html'
elif mon == 4 or mon == 10:
cmd = "cp /data/mta_www/mta_grat/EdE/heg_all.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/EdE/meg_all.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_grat/EdE/leg_all.gif " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/twist_plot.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY4.html'
elif mon == 5 or mon == 11:
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd3.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd5.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd7.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_pos.html " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_width.html " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_cnt.html " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY5.html'
elif mon == 6 or mon == 12:
cmd = "cp /data/mta_www/mta_sim_twist/Plots/I-1.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/S-2.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-I-2.png " + odir
os.system(cmd)
cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-S-2.png " + odir
os.system(cmd)
file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY6.html'
#
#--- create clean acis and hrc exposure maps using ds9
#
run_exposure_maps(lyear, cmon)
#--------------------------------------------------
#--- read the template and substitute the contents
#--------------------------------------------------
fx = open(file, 'r')
text = fx.read()
fx.close()
#
#--- substitute values
#
fmonth_list = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
fmon = fmonth_list[mon-1]
text = text.replace('#Month#', fmon) #--- full month name e.g. May
text = text.replace('#YEAR#', lyear) #--- full year nane e.g. 2016
text = text.replace('#SMON#', lmon) #--- short month, e.g. Jan
text = text.replace('#LSMON#', lmon.lower())#--- short lower month, e.g. jan
line = 'rad_use_' + lmonyr + '.png'
text = text.replace('#RADUSE#', line) #--- e.g., rad_use_oct16.png
line = cmon + '_' + lyear
text = text.replace('#LMONYR#', line) #--- mon_year e.g. 07_2016
#
#--- acis exposure tables
#
os.system("cd /data/mta/Script/Exposure/Exc/; /data/mta/Script/Exposure/Scripts/ACIS_Scripts/acis_dose_monthly_report.py")
line = '/data/mta/Script/Exposure/Exc/monthly_diff_' + cmon + '_' + lyear
f = open(line, 'r')
dout = f.read()
f.close()
text = text.replace('#ACISMON#', dout) #--- acis monthly dose
line = '/data/mta/Script/Exposure/Exc/monthly_acc_' + cmon + '_' + lyear
f = open(line, 'r')
dout = f.read()
f.close()
text = text.replace('#ACISCMON#', dout) #--- acis cumulative dose
#
#--- last 12 months exposure maps
#
byear = year -1
text = past_data_entry(1, 0, mon, byear, text)
text = past_data_entry(2, 3, mon, byear, text)
text = past_data_entry(3, 6, mon, byear, text)
text = past_data_entry(4, 9, mon, byear, text)
#
#---- acis focal
#
fx = open('/data/mta/Script/Month/FOCAL/Plots/month_avg', 'r')
data = [line.strip() for line in fx.readlines()]
fx.close()
atemp = re.split(':', data[0])
btemp = re.split('\+\/\-', atemp[1])
ft = round(float(btemp[0]), 2)
ferr = round(float(btemp[1]), 2)
text = text.replace('#FT#', str(ft))
text = text.replace('#FERR#', str(ferr))
atemp = re.split(':', data[1])
btemp = re.split('\+\/\-', atemp[1])
fw = round(float(btemp[0]), 2)
werr = round(float(btemp[1]), 2)
text = text.replace('#FW#', str(fw))
text = text.replace('#WERR#', str(werr))
line = 'erad_' + lmonyr + '.gif' #--- erand_nov16.gif
text = text.replace('#ERAND#', line)
#
#--- acis sib
#
yearmon = str(year) +'_' + cmon
text = text.replace('#YEARMON#', yearmon) #--- e.g. 2015_03
#
#--- hrc i monthly dose
#
line = hrc_monthly_report('HRCI', cmon, lmon, lyear)
text = text.replace('#HRCIDOSE#', line) #--- HRC I DIFF
#
#--- hrc cumulative
#
line = hrc_cumulative_report('HRCI', cmon, lmon, lyear)
text = text.replace('#CHRCIDOSE#', line) #--- HRC I Cumulative
#
#--- hrc s monthly dose
#
line = hrc_monthly_report('HRCS', cmon, lmon, lyear)
text = text.replace('#HRCSDOSE#', line) #--- HRC S DIFF
#
#--- hrc cumulative
#
line = hrc_cumulative_report('HRCS', cmon, lmon, lyear)
text = text.replace('#CHRCSDOSE#', line) #--- HRC S Cumulative
#
#--- IRU
#
text = text.replace("#SMONYR#", lmonyr) #--- e.g., nov16
#
#--- envelope trending
#
line = get_envelope_trending(mon, odir)
text = text.replace("#ENVTREND#", line)
#
#--- critical trends which are reported on Mar, Jun, Sep, and Dec
#
if mon in [3, 6, 9, 12]:
text = run_critical_trend(text)
#
#--- Monthly Trend Reports
#
text = run_month_trend(mon, text)
#
#--- last index
#
line = create_index(year, mon)
text = text.replace('#YINDEX#', line)
#
#--- finally print out the report
#
ofile = odir + "MONTHLY.html"
fo = open(ofile, 'w')
fo.write(text)
fo.close()
#
#--- chnage permission and owners
#
cmd = 'chmod 755 ' + odir + '/*'
os.system(cmd)
cmd = 'chgrp -R mtagroup ' + odir + '/*'
os.system(cmd)
def create_sim_temp_plots(start, stop):
"""
control function to run sim temperature trending plots etc
input: start --- starting year
stop --- stopping year
if they are the same, it will cerate the plot for the year
else, it will create the entire range (from 1999 to current)
output: plots in <web_dir>/Plots/
sim_temp_<year>.png
sim)translation_<year>.png
"""
#
#--- check whether to create full range or indivisual year plots
#
if stop - start > 1:
fname = web_dir + 'Plots/sim_temp_fullrange.png'
oname = web_dir + 'Plots/sim_translation_fullrange.png'
pname = web_dir + 'Plots/pitchangle_fullrange.png'
title = str(start) + ' - ' + str(stop-1)
yind = 0
else:
fname = web_dir + 'Plots/sim_temp_' + str(start) + '.png'
oname = web_dir + 'Plots/sim_translation_' + str(start) + '.png'
pname = web_dir + 'Plots/pitchangle_' + str(start) + '.png'
title = 'Year: ' + str(start)
yind = 1
if tcnv.isLeapYear(start) == 1:
base = 366
else:
base = 365
#
#--- read sim temperature data file
#
file = data_dir + 'tsc_temps.txt'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
x = []
ts = []
te = []
delta = []
steps = []
for ent in data[1:]:
atemp = re.split('\s+', ent)
try:
time = convert_to_fyear(atemp[0])
if time < start:
continue
if time > stop:
break
if yind == 1:
time = (time - start) * base
t1 = float(atemp[2])
t2 = float(atemp[3])
t3 = t2 - t1
t4 = float(atemp[5])
if t1 > 60 or t1 < -40:
continue
if t2 > 60 or t2 < -40:
continue
x.append(time) #--- time in year or yday
ts.append(t1) #--- sim temperature at the beginning
te.append(t2) #--- sim temperature at the ending
delta.append(t3) #--- the sim temperature difference
steps.append(t4) #--- the number of steps
except:
continue
#
#--- run moving average
#
mvstep = 30
[mxs, mys] = smooth_data(x, ts, mvstep)
if len(mxs) == 0:
skip = 1
else:
skip = 0
[mxe, mye] = smooth_data(x, te, mvstep)
if len(mxe) == 0:
skip = 1
else:
skip = 0
#
#--- read pitch angle data
#
file = '/data/mta/DataSeeker/data/repository/orb_angle.rdb'
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
stime = []
angle = []
m = 0
for ent in data[2:]:
atemp = re.split('\s+', ent)
try:
time = tcnv.sectoFracYear(float(atemp[0]))
if time < start:
continue
if time > stop:
break
if yind == 1:
time = (time - start) * base
#
#--- just take the data every 20 mins or so (data is taken every 300 sec)
#
if m % 4 == 0:
stime.append(time)
angle.append(float(atemp[1]))
m += 1
except:
continue
#
#--- if yind == 0, full range, otherwise, year plot
#
if yind == 0:
xmin = start
xmax = stop
tdff = 1.0e-5
else:
xmin = 1
xmax = base
tdff = 3.5e-3
#
#--- match delta and angle --- not used
#
## angle2 = []
## js = 0
## for k in range(0, len(x)):
## angle2.append(0)
## for j in range(js, len(stime)):
## if x[k] < stime[j]:
## angle2[k] = angle[j]
## break;
## else:
## if j >= len(stime) -1:
## angle2[k] = angle[-1]
## break;
## if x[k] >= stime[j] and x[k] <= stime[j+1]:
## angle2[k] = angle[j]
## js = j
## break
## else:
## continue
#
#--- plot time tred of sim temperature
#
plot_sim_temp_data(xmin, xmax, x, ts, te, mxs, mys, mxe, mye, delta, stime, angle, fname, yind, title, skip)
#
#--- plot translation step - delta sim temperature
#
plot_step_delta(steps, delta, oname, title)
def plot_data():
"""
plotting moving aerage of peak temperature and peak width for the entire period
input: none, but read from focal_temp in the same directory. this must be updated
before compute.
output: focal_temp_plot.png
"""
#
#--- read data
#
f = open(fdata, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
time = []
focal = []
width = []
for ent in data:
atemp = re.split('\s+', ent)
try:
year = float(atemp[0])
ydate = float(atemp[1])
if tcnv.isLeapYear(year) == 1:
base = 366
else:
base = 365
fyear = year + ydate/base
time.append(fyear)
focal.append(float(atemp[2]))
width.append(float(atemp[3]))
except:
continue
#
#--- create 10 day moving average
#
[mv_time, mv_temp, mv_width] = moving_avg(time, focal, width, period=10)
#
#--- set a few parameters for plotting
#
plt.close('all')
mpl.rcParams['font.size'] = 18
mpl.rcParams['font.weight'] = 'bold'
plt.subplots_adjust(hspace=0.08)
#
#--- set plotting ranges (x axis in year)
#
xmin = 2000
xmax = int(max(mv_time)) + 1
if xmax - max(mv_time) < 0.3:
xmax += 1
#
#--- focal temperature plot (top panel)
#
a1 = plt.subplot(211)
ymin = -120
ymax = -95
yname = 'Focal Temperature (C)'
title = "Focal Temperature (10 Day Moving Average)"
plot_panel(a1, mv_time, mv_temp, xmin, xmax, ymin, ymax, yname, title)
#
#--- peak width plot (bottom panel)
#
a2 = plt.subplot(212)
ymin = 0
ymax = 2.0
yname = 'Width (days)'
title = "Peak Width (10 Day Moving Average)"
plot_panel(a2, mv_time, mv_width, xmin, xmax, ymin, ymax, yname, title)
#
#--- label x axis only on the bottom one
#
line = a1.get_xticklabels()
for label in line:
label.set_visible(False)
a2.set_xlabel('Time (Year)', fontweight='bold')
#
#--- changing frame line width
#
for axis in ['top','bottom','left','right']:
a1.spines[axis].set_linewidth(3)
a2.spines[axis].set_linewidth(3)
#
#---- set plotting area size
#
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(20.0,10.0)
#
#--- print out the plot
#
outname = './Plots/focal_temp_plot.png'
plt.savefig(outname, format='png', dpi=300)
plt.close('all')
