def kplookup(ifile):
"""
look up kp value and append to the data
input: ifile --- input data file name
output: ofile --- output data file name; <ifile>0
"""
#
#--- read data and separate into column data; stime is Chandra time
#
data = mcf.read_data_file(ifile)
cdata = mcf.separate_data_to_arrays(data)
#
#--- make sure that the data is accending order in time
#
stime = numpy.array(cdata[0])
ndata = numpy.array(data)
ind = numpy.argsort(stime)
stime = list(stime[ind])
ndata = list(ndata[ind])
#
#--- read the current kp data
#
kpdata = mcf.read_data_file(kp_file)
[csec, ckp] = mcf.separate_data_to_arrays(kpdata)
#
#--- compare them and match kp to the data
#
dlen = len(ndata)
klen = len(ckp)
line = ''
mv = 1
lpos = 0
for k in range(0, dlen):
for m in range(mv, klen):
if (stime[k] >= csec[m - 1]) and (stime[k] < csec[m]):
line = line + ndata[k] + '\t' + "%1.2f" % float(
ckp[m - 1]) + '\n'
lpos = k
mv -= 5
if mv < 0:
mv = 0
break
elif stime[k] < csec[m - 1]:
break
elif stime[k] > csec[m]:
mv += 1
continue
#
#--- fill up the data with the last kp value estimated
#
for k in range(lpos + 1, dlen):
line = line + ndata[k] + '\t' + "%1.2f" % float(ckp[-1]) + '\n'
#
#--- write out the ifile with matched kp value (a file name has an extra "0" at the end)
#
ofile = ifile + '0'
with open(ofile, 'w') as fo:
fo.write(line)
def rearrangeData(data):
"""
separate a table data into time and 4 quad data array data sets
Input: data --- input table data
Output: xSets --- a list of lists of x values
ySets --- a list of lists of y values
yErrs --- a list of lists of y errors
"""
xSets = []
ySets = []
yErrs = []
coldata = mcf.separate_data_to_arrays(data)
stime = convert_time_list(coldata[0]) #---- time in ydate
#
#--- go around each quad data
#
for i in range(1, 5):
xSets.append(stime)
data = []
error = []
for ent in coldata[i]:
atemp = re.split('\+\-', ent)
data.append(float(atemp[0]))
error.append(float(atemp[1]))
ySets.append(data)
yErrs.append(error)
return (xSets, ySets, yErrs)
def read_in_data(ifile, col=1):
"""
read the data and return the cleaned up arrays of time and data
input: ifile --- a file name
col --- a column position of the data set; default: 1
we assume that the first column (0) is time in seconds from 1998.1.1
output: t_array --- an array of time
d-array --- an array of data
"""
data = mcf.read_data_file(ifile)
if len(data) < 1:
return [[], []]
data_set = mcf.separate_data_to_arrays(data)
t_array = numpy.array(data_set[0])
d_array = numpy.array(data_set[col])
#
#--- get rid of nan data points
#
idx = ~numpy.isnan(d_array)
t_array = t_array[idx]
d_array = d_array[idx]
#
#--- get rind of bad data (usually -999.0)
#
idx = d_array > -10
t_array = t_array[idx]
d_array = d_array[idx]
return [t_array, d_array]
def create_recent_one_year_plot():
"""
create most recent one year trending plots
input: none, but read from <data_dir>/<slot name>_<#>
output: <web_dir>/Plots/<slot name>_<#>_recent_1yr.png
"""
for m in range(0, 3):
ifile = data_dir + data_list[m]
#
#--- read data, separate into column data, and select the data only between time period specified
#
data_set = mcf.read_data_file(ifile)
data_set = mcf.separate_data_to_arrays(data_set)
#
#--- find the data starting time (one year before the last data point)
#
tstop = data_set[0][-1]
tstart = tstop - oneyear
data_set = select_data_for_time_period(data_set, tstart, tstop)
#
#--- save time in fractional year
#
time_list = []
for tval in data_set[0]:
time_list.append(mcf.chandratime_to_fraq_year(tval))
for k in range(1, len(data_set)):
#
#--- acacent_mtatr hold 2 sets of data
#
if m == 2:
if k < 9:
y_name = slot_name[m] + 'ynea_' + str(k - 2)
else:
y_name = slot_name[m] + 'znea_' + str(k - 9)
else:
y_name = slot_name[m] + '_' + str(k - 2)
out_name = web_dir + 'Plots/' + y_name + '_recent_1yr.png'
[t_list, d_list] = drop_nan(time_list, data_set[k])
if len(t_list) < 1:
cmd = 'cp ' + house_keeping + 'no_data.png' + ' ' + out_name
os.system(cmd)
continue
xlabel = 'Time (year)'
plot_data(t_list, d_list, xlabel, y_name, out_name, xs=1)
def read_ede_data(infile):
"""
read ede data file
input: infile --- input file name
output: a list of:
idata[0]: year
idata[1]: obsid
idata[2]: energy
idata[3]: fwhm
idata[4]: denergy
idata[5]: error
idata[6]: order
idata[7]: cnt
idata[8]: roi_cnt
idata[9]: acf
idata[10]: acf_err
idata[11]: links
"""
dfile = data_dir + infile
data = mcf.read_data_file(dfile)
idata = mcf.separate_data_to_arrays(data, com_out='#')
year = numpy.array(idata[0]).astype(int)
obsid = []
for ent in idata[1]:
try:
val = str(int(float(ent)))
except:
val = ent
obsid.append(val)
energy = numpy.array(idata[2])
fwhm = idata[3]
denergy = numpy.array(idata[4])
error = idata[5]
order = list(numpy.array(idata[6]).astype(int))
cnt = list(numpy.array(idata[7]).astype(int))
roi_cnt = idata[8]
acf = idata[9]
acf_err = idata[10]
links = idata[11]
return [
year, obsid, energy, fwhm, denergy, error, order, cnt, roi_cnt, acf,
acf_err, links
]
def create_full_lange_plot():
"""
create full range trending plots
input: none, but read from <data_dir>/<inst>_month
output: <web_dir>/Plots/<inst>_full.png
"""
for m in range(0, 3):
ifile = data_dir + data_list[m] + '_month'
#
#--- read data, separate into column data
#
data_set = mcf.read_data_file(ifile)
data_set = mcf.separate_data_to_arrays(data_set)
#
#--- save time in fractional year format
#
time_list = []
for ent in data_set[1]:
atemp = re.split(':', ent)
tval = float(atemp[0]) + float(atemp[1]) / 12.0
time_list.append(tval)
for k in range(2, len(data_set)):
#
#--- acacent_mtatr hold 2 sets of data
#
if m == 2:
if k < 9:
y_name = slot_name[m] + 'ynea_' + str(k - 2)
else:
y_name = slot_name[m] + 'znea_' + str(k - 9)
else:
y_name = slot_name[m] + '_' + str(k - 2)
out_name = web_dir + 'Plots/' + y_name + '_full.png'
[t_list, d_list] = drop_nan(time_list, data_set[k])
if len(t_list) < 1:
cmd = 'cp ' + house_keeping + 'no_data.png' + ' ' + out_name
os.system(cmd)
continue
xlabel = 'Time (year)'
plot_data(t_list, d_list, xlabel, y_name, out_name)
def get_sunangle_data(tstart, tstop):
"""
read sunangle data
input: tstart --- starting time in seconds from 1998.1.1
tstop --- stopping time in seconds from 1998.1.1
output stime --- a list of time
suncent --- a list of sun center angle
"""
#
#--- sun center angle
#
data = mcf.read_data_file(orb_data)
data = mcf.separate_data_to_arrays(data)
#
#--- first two rows are headers; skip them
#
stime = data[0][2:]
suncent = data[1][2:]
istart = -999
istop = len(stime)
for k in range(0, len(stime)):
if (istart < 0) and (stime[k] > tstart):
istart = k
elif stime[k] > tstop:
istop = k
break
if istart < 0:
istart = 0
try:
stime = stime[istart:istop]
suncent = suncent[istart:istop]
except:
exit(1)
if len(stime) < 1:
exit(1)
return [stime, suncent]
def isolateData(ccd, elm, quad, dir_set):
"""
separate a table data into arrays of data
Input: ccd --- ccd #
elm --- name of element
quad --- quad #
dir_set --- a list of data directories
Output: xSets --- a list of lists of x values
ySets --- a list of lists of y values
eSets --- a list of lists of y errors
"""
xSets = []
ySets = []
eSets = []
for idir in dir_set:
ifile = data_dir + idir + '/' + elm + '_ccd' + str(ccd)
data = mcf.read_data_file(ifile)
if len(data) < 1:
xSets.append([])
ySets.append([])
eSets.append([])
continue
#
#--- separeate a table into each column array
#
coldata = mcf.separate_data_to_arrays(data)
#
#--- convert time into fractional year (a list of time)
#
stime = convert_time_list(coldata[0])
xSets.append(stime)
#
#--- the data part come with cti +- error. drop the error part
#
[ydata, yerr] = separateErrPart(coldata[quad + 1])
ySets.append(ydata)
eSets.append(yerr)
return [xSets, ySets, eSets]
def read_zero_data(infile):
"""
read data file and return lists of times and values
input: infile --- data file name
output: t_list --- a list of time data
c1_list --- a list of data (sky_x)
c2_list --- a list of data (sky_y)
c3_list --- a list of data (chip_x)
c4_list --- a list of data (chip_y)
"""
infile = data_dir + infile
data = mcf.read_data_file(infile)
dout = mcf.separate_data_to_arrays(data)
t_list = [mcf.chandratime_to_fraq_year(x) for x in dout[0]]
c1_list = dout[1]
c2_list = dout[2]
c3_list = dout[3]
c4_list = dout[4]
return [t_list, c1_list, c2_list, c3_list, c4_list]
def read_exp_stat_data(ifile):
"""
read a data file and seven column data lists
input: ifile --- input file
output: a list of lists of:
date, mean, min, max, s1, s2, s3
"""
data = mcf.read_data_file(ifile)
out = mcf.separate_data_to_arrays(data)
# avg = out[2]
# smin = out[4]
# smax = out[6]
# s1 = out[8]
# s2 = out[9]
# s3 = out[10]
##
##--- replace 'NA' into 0
##
# avg = [0 if str(x).lower() == 'na' else x for x in avg]
# smin = [0 if str(x).lower() == 'na' else x for x in smin]
# smax = [0 if str(x).lower() == 'na' else x for x in smax]
# s1 = [0 if str(x).lower() == 'na' else x for x in s1]
# s2 = [0 if str(x).lower() == 'na' else x for x in s2]
# s3 = [0 if str(x).lower() == 'na' else x for x in s3]
mout = []
for ent in out:
fixed = [0 if str(x).lower() == 'na' else x for x in ent]
mout.append(fixed)
#
#--- time is in fractional year format
#
date = []
for k in range(0, len(mout[0])):
time = float(mout[0][k]) + float(mout[1][k])/12.0 + 0.5
date.append(time)
rout = [date]
rout = rout + mout
return rout
def create_gyro_drift_ind_page():
"""
create gryro drift movment html pages
input: none
output: <web_dir>/Individual_plots/<GRATING>_<ACTION>/<catg>_<grating>_<action>.html
"""
#
#--- read template
#
tname = house_keeping + 'drift_plot_template'
with open(tname, 'r') as f:
template = f.read()
#
#--- go through all data set to create several sub html pages related to indivisual fitting resluts
#
for catg in catg_list:
for grating in ['hetg', 'letg']:
for action in ['insr', 'retr']:
ifile = data_dir + 'gyro_drift_' + catg + '_' + grating + '_' + action
data = mcf.read_data_file(ifile)
data = mcf.separate_data_to_arrays(data, com_out='#')
create_html_page_for_movement(catg, grating, action, data,
template)
def cocochan(ifile):
"""
convert Chandra ECI linear coords to GSE, GSM coords
input: ifile --- a data file name with:
t, x, y, z, vx, vy, vz, fy, mon, day, hh, mm, ss, kp
output: line --- data line with:
t, x, y, z, xsm, ysm, zsm, lid
lid = 1 if spacecraft is in solar wind
lid = 2 if spacecraft is in magnetosheath
lid = 3 if spacecraft is in magnetosphere
Note: this python script is converted from Robert Cameron (2001) f77 program
cocochan.f and crmflx_**.f which came with geopack.f
python version of geopack: https://pypi.org/project/geopack/
"""
#
#--- read data
#
out = mcf.read_data_file(ifile)
#
#--- separate data and convert into column data
#
try:
[t, x, y, z, vx, vy, vz, fy, mon, day, hh, mm, ss,
kp] = mcf.separate_data_to_arrays(out)
except:
[t, x, y, z, vx, vy, vz, fy, mon, day, hh, mm,
ss] = mcf.separate_data_to_arrays(out)
kp = [1] * len(t)
#
#--- compute ut in seconds from 1970.1.1
#
uts = ut_in_secs(fy[0], mon[0], day[0], hh[0], mm[0], ss[0])
#
#--- usually this need to be run only once but it seems better to run every time;
#--- value is kept to be used by other geopack functions
#
psi = geopack.recalc(uts)
line1 = ''
line2 = ''
line3 = ''
for k in range(0, len(t)):
#
#--- convert position to km
#
xs = x[k] / 1.0e3
ys = y[k] / 1.0e3
zs = z[k] / 1.0e3
r = math.sqrt(xs * xs + ys * ys + zs * zs)
uts = ut_in_secs(fy[k], mon[k], day[k], hh[k], mm[k], ss[k])
psi = geopack.recalc(uts)
#
#--- convert the coordinates into gsm and gse
#
[xgsm, ygsm, zgsm, xgm, ygm, zgm, xge, yge, zge,
lid] = compute_gsm(xs, ys, zs, kp[k])
#
#--- convert to special coordinates
#
# [mr, mt, mp] = convert_to_special_coords(xgsm, ygsm, zgsm)
# [er, et, ep] = convert_to_special_coords(xge, yge, zge)
line1 = line1 + '%11.1f' % t[k]
line1 = line1 + '\t%10.2f' % r
line1 = line1 + '\t%10.2f' % xs
line1 = line1 + '\t%10.2f' % ys
line1 = line1 + '\t%10.2f' % zs
line1 = line1 + '\t%10.2f' % xgsm
line1 = line1 + '\t%10.2f' % ygsm
line1 = line1 + '\t%10.2f' % zgsm
line1 = line1 + '\t\t%1d' % lid
line1 = line1 + '\n'
return line1
def plot_gyro_drift_trends():
"""
plot gyro drinf trends
input: none, but read from <data_dir>/gyro_drift_<catg>_<grating>_<action>
output: <web_dir>/Trending_plots/<catg>_<grating>_<action>_<position>.png
<postion> can be: before, during, after, db_ratio, ad_ratio, or ba_ratio
"""
#
#--- set the end of the x plot range
#
lyear = time.strftime("%Y", time.gmtime())
lyear = int(lyear) + 1
#
#--- go through all data set to create several sub html pages related to indivisual fitting resluts
#
for catg in catg_list:
for grating in ['hetg', 'letg']:
for action in ['insr', 'retr']:
ifile = data_dir + 'gyro_drift_' + catg + '_' + grating + '_' + action
data = mcf.read_data_file(ifile)
data = mcf.separate_data_to_arrays(data, com_out='#')
data = select_data(data)
#
#--- convert time into fractional year
#
x = convert_time_to_year(data[0])
#
#--- std of before the movement
#
y = get_std_part(data[1])
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_before.png'
plot_data(x, y, 1999, lyear, 0.0, 2.0, 'Time (year)', catg, out)
#
#--- std of during the movement
#
y = get_std_part(data[2])
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_during.png'
plot_data(x, y, 1999, lyear, 0.0, 2.0, 'Time (year)', catg, out)
#
#--- std of after the movement
#
y = get_std_part(data[3])
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_after.png'
plot_data(x, y, 1999, lyear, 0.0, 2.0, 'Time (year)', catg, out)
#
#--- ratio of stds of before and during the movement
#
y = data[4]
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_bd_ratio.png'
plot_data(x, y, 1999, lyear, 0.0, 4.0, 'Time (year)', catg, out)
#
#--- ratio of stds of after and during the movment
y = data[5]
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_ad_ratio.png'
plot_data(x, y, 1999, lyear, 0.0, 4.0, 'Time (year)', catg, out)
#
#--- ratio of stds of before and after the movement
#
y = data[6]
out = web_dir + 'Trending_plots/' + catg + '_' + grating + '_' + action + '_ba_ratio.png'
plot_data(x, y, 1999, lyear, 0.0, 4.0, 'Time (year)', catg, out)
def create_year_long_plot(year):
"""
create one year long plots for the given year. if the year is not given, use this year
input: year --- year
also read from <data_dir>/<slot name>_<#>
output: <web_dir>/Plots/<year>/<slot name>_<#>.png
"""
#
#--- if the year is not given, use this year, except the first 10 days of year
#--- create the last year's plot
#
if year == '':
year = int(float(time.strftime('%Y', time.gmtime())))
mday = int(float(time.strftime('%j', time.gmtime())))
if mday < 5:
year -= 1
#
#--- set starting and stopping time of the data interval
#
tstart = str(year) + ':001:00:00:00'
tstart = Chandra.Time.DateTime(tstart).secs
tstop = tstart + oneyear
if mcf.is_leapyear(year):
tstop += 86400.0
for m in range(0, 3):
#
#--- read data, separate into column data, and select the data only between time period specified
#
ifile = data_dir + data_list[m]
data_set = mcf.read_data_file(ifile)
data_set = mcf.separate_data_to_arrays(data_set)
data_set = select_data_for_time_period(data_set, tstart, tstop)
#
#--- save time in day of year
#
time_list = []
for tval in data_set[0]:
time_list.append(mcf.chandratime_to_yday(tval))
#
#--- start plotting each slot
#
for k in range(1, len(data_set)):
#
#--- acacent_mtatr hold 2 sets of data
#
if m == 2:
if k < 9:
y_name = slot_name[m] + 'ynea_' + str(k - 2)
else:
y_name = slot_name[m] + 'znea_' + str(k - 9)
else:
y_name = slot_name[m] + '_' + str(k - 1)
out_name = y_name + '.png'
[t_list, d_list] = drop_nan(time_list, data_set[k])
xlabel = 'Day of Year (Year: ' + str(year) + ')'
odir = web_dir + 'Plots/' + str(year) + '/'
if not os.path.isdir(odir):
cmd = 'mkdir -p ' + odir
os.system(cmd)
out_name = odir + out_name
if len(t_list) < 1:
cmd = 'cp ' + house_keeping + 'no_data.png' + ' ' + out_name
os.system(cmd)
continue
plot_data(t_list, d_list, xlabel, y_name, out_name)
def separate_data(data):
"""
separate each element to a set of arrays
Input: data --- data sets which contains:
<start><quad#>+-<err> <obsid> <stop> <span> <temperature>
Output: quad# --- cti value for the quad
err# --- error value for the quad
start --- start time
stop --- stop time
obsid --- obsid
span --- time span in seconds
temp --- focal temperature
del_tep --- temperature - 119.7
"""
#
#--- initialize
#
cti = [[] for x in range(4)]
err = [[] for x in range(4)]
start = []
stop = []
obsid = []
temp = []
sigm = []
tmin = []
tmax = []
secs = []
sece = []
del_temp = []
#
#--- separate cti values and error values
#
try:
outarray = mcf.separate_data_to_arrays(data)
chk = 1
except:
chk = 0
if chk > 0:
for i in range(1, 5):
for ent in outarray[i]:
atemp = re.split('\+\-', ent)
cti[i - 1].append(atemp[0])
err[i - 1].append(atemp[1])
start = outarray[0]
obsid = outarray[5]
stop = outarray[6]
temp = outarray[7]
sigm = outarray[8]
tmin = outarray[9]
tmax = outarray[10]
secs = outarray[11]
sece = outarray[12]
for ent in temp:
val = float(ent) + 119.7
del_temp.append(val)
return (cti[0], cti[1], cti[2], cti[3], err[0], err[1], err[2], err[3], start, stop,\
obsid, temp, sigm, tmin, tmax, secs, sece, del_temp)
def create_monthly_average(fdname, mdname, tyear, tmonth):
"""
create monthly averaged dataset from the full data set
input: fdname --- a name of full data set
mdname --- a name of monthly average data set
tyear --- the stopping year of the data collection
tmonth --- the stopping month of the data collection
output: mdname --- updated monthly averaged data set
"""
#
#--- read full dataset
#
data = mcf.read_data_file(fdname)
data_set = mcf.separate_data_to_arrays(data)
dlen = len(data_set) #--- # of columns in the full data set
mlen = dlen + 1 #--- # of columns in the averaged data set
#
#--- read the monthly averaged data
#
data = mcf.read_data_file(mdname)
if len(data) > 0:
mon_data = mcf.separate_data_to_arrays(data)
#
#--- find the last entry date and set starting time to the beginning of the month of that time period
#--- this is because the data of that last entties could be incomplete
#
atemp = re.split('\s+', data[-1])
ltime = float(atemp[0])
ltime = Chandra.Time.DateTime(ltime).date
atemp = re.split('\.', ltime)
ltime = time.strftime('%Y:%m', time.strptime(atemp[0],
'%Y:%j:%H:%M:%S'))
atemp = re.split(':', ltime)
syear = int(float(atemp[0]))
smonth = int(float(atemp[1]))
#
#--- if this is the first time running this script, start from the empty data set
#
else:
mon_data = []
for k in range(0, mlen):
mon_data.append([])
syear = 1999
smonth = 8
#
#--- some initializations
#
tlen = len(data_set[0]) #--- length of the each data set
save = [] #--- a list of lists to save mean of each month
slp_t = [] #--- a list of time related to slope list
slp_y = [] #--- a list of time related to slope list in <yyyy>:<mm>
slp_s = [] #--- a list of lists to save the slope of each month
std_s = [] #--- a list of lists to save std of each month
tsave = [] #--- a list of lists to save time for each value
sums = [] #--- a list of lists to save the value for one month
#
#--- save which save monthly average, will hold the data from beginning (1999:08)
#--- but slope and the std hold only the newly computed part only
#
for k in range(0, mlen):
save.append([])
slp_s.append([])
std_s.append([])
for k in range(0, dlen):
sums.append([])
tsave.append([])
dpos = 0 #--- index on the full data set
ipos = 0 #--- index on the monthly data set
test = 0
#
#--- before the data collection period starts, use the data from the monthly data set
#
for year in range(1999, tyear + 1):
for month in range(1, 13):
if year < syear:
for k in range(0, mlen):
try:
save[k].append(mon_data[k][ipos])
except:
pass
ipos += 1
continue
elif (year == syear) and (month < smonth):
for k in range(0, mlen):
try:
save[k].append(mon_data[k][ipos])
except:
pass
ipos += 1
continue
elif (year == tyear) and (month > tmonth):
break
#
#--- data gathering of each month from syear:smonth to tyear:tmonth starts here
#
#if (year == syear) and (month == smonth):
# ldate = str(year) + ':' + mcf.add_leading_zero(month)
# slp_y.append(ldate)
# continue
nyear = year
nmonth = month + 1
if nmonth > 12:
nmonth = 1
nyear += 1
#
#--- set start, mid point, and stop time in seconds from 1998.1.1
#
pstart = get_ctime(year, month, 1)
pmid = get_ctime(year, month, 15)
pstop = get_ctime(nyear, nmonth, 1)
#
#--- go through the main data
#
for m in range(dpos, tlen):
if data_set[0][m] < pstart:
continue
#
#--- data gethering of the period finished, compute averages
#
elif data_set[0][m] > pstop:
#
#--- for the time use 15th of the month in seconds from 1998.1.1 format
#
save[0].append(pmid)
#
#--- second column is <yyyy>:<mm>
#
ldate = str(year) + ':' + mcf.add_leading_zero(month)
save[1].append(ldate)
slp_t.append(pmid)
slp_y.append(ldate)
#
#--- now the slot data parts
#
for n in range(1, dlen):
#
#--- if there are data, make sure that they are not 'nan' before taking a mean
#
if len(sums[n]) > 0:
[t_list, v_list] = clean_up_list(tsave[n], sums[n])
if len(t_list) > 0:
save[n + 1].append(numpy.mean(v_list))
#
#--- compute linear fitting on the data for the month and also compute the std of the data
#
if len(t_list) > 3:
t_list = convert_to_ydate_list(t_list)
[aa, bb,
delta] = rlf.least_sq(t_list, v_list)
else:
bb = -999
slp_s[n + 1].append(bb)
std_s[n + 1].append(numpy.std(v_list))
#
#--- if there is no valid data, use -999
#
else:
save[n + 1].append(-999)
slp_s[n + 1].append(-999)
std_s[n + 1].append(-999)
#
#--- initialize the sums/tsave lists for the next round
#
try:
val = float(data_set[n][m])
sums[n] = [val]
tsave[n] = [data_set[0][m]]
except:
sums[n] = []
tsave[n] = []
continue
#
#--- no data during the period; just put -999
#
else:
save[n + 1].append(-999)
slp_s[n + 1].append(-999)
std_s[n + 1].append(-999)
sums[n] = []
tsave[n] = []
try:
val = float(data_set[n][m])
sums[n] = [val]
tsave[n] = [data_set[0][m]]
except:
sums[n] = []
tsave[n] = []
continue
#
#--- a few other initialization for the next round
#
sums[0] = [data_set[0][m]]
tsave[0] = [data_set[0][m]]
dpos = m + 1 #--- where to start reading the big data
lyear = year #--- keep year and month for the later use
lmonth = month
test = 1
break
#
#--- the time is between the period. accumulate the data
#
else:
for n in range(1, dlen):
try:
val = data_set[n][m]
if str(val).lower() in ['nan', 'na']:
continue
val = float(val)
#
#--- make sure that the value is reasonable
#
if val < 100 and val > -100:
sums[n].append(val)
tsave[n].append(data_set[0][m])
except:
continue
lyear = year
lmonth = month
test = 0
#
#--- the last part may not be a complete month data
#
if test == 0:
for n in range(1, dlen):
test += len(sums[n])
if test > 0:
save[0].append(pmid)
ldate = str(lyear) + ':' + mcf.add_leading_zero(lmonth)
save[1].append(ldate)
slp_t.append(pmid)
slp_y.append(ldate)
for n in range(1, dlen):
if len(sums[n]) > 0:
[t_list, v_list] = clean_up_list(tsave[n], sums[n])
if len(t_list) > 0:
save[n + 1].append(numpy.mean(v_list))
if len(t_list) > 3:
t_list = convert_to_ydate_list(t_list)
[aa, bb, delta] = rlf.least_sq(t_list, v_list)
else:
bb = -999.0
slp_s[n + 1].append(bb)
std_s[n + 1].append(numpy.std(v_list))
else:
save[n + 1].append(-999)
slp_s[n + 1].append(-999)
std_s[n + 1].append(-999)
else:
save[n + 1].append(-999)
slp_s[n + 1].append(-999)
std_s[n + 1].append(-999)
#
#--- update the mean monthly data set
#
line = ''
for k in range(0, len(save[0])):
#
#--- prevent a dupulicated line
#
if k != 0:
if save[0][k] == save[0][k - 1]:
continue
line = line + '%d\t' % save[0][k]
line = line + save[1][k] + '\t'
for n in range(2, mlen):
line = line + format_line(save[n][k])
line = line + '\n'
with open(mdname, 'w') as fo:
fo.write(line)
#
#--- update the slope/std monthly data set
#
line = ''
for k in range(0, len(slp_t)):
line = line + '%d\t' % slp_t[k]
line = line + slp_y[k] + '\t'
for n in range(2, mlen):
line = line + format_line(slp_s[n][k])
for n in range(2, mlen):
line = line + format_line(std_s[n][k])
line = line + '\n'
#
#--- since the slope data are saved only the new part, read the old data
#--- and find the spot whether the data are updated.
#
sdname = mdname + '_slope'
data = mcf.read_data_file(sdname)
try:
btemp = re.split(':', slp_y[0])
syear = int(float(btemp[0]))
smon = int(float(btemp[1]))
aline = ''
for ent in data:
atemp = re.split('\s+', ent)
btemp = re.split(':', atemp[1])
cyear = int(float(btemp[0]))
cmon = int(float(btemp[1]))
if cyear == syear and cmon == smon:
break
else:
aline = aline + ent + '\n'
except:
aline = ''
aline = aline + line
with open(sdname, 'w') as fo:
fo.write(aline)
def runcrm(ifile=''):
"""
calculate CRM proton flux for Chandra ephemeris
read from a file, for all 28 possible values of Kp.
input: ifile --- input ephemeris file, e.g., 'PE.EPH.gsme_in_Re_short'
output: <crm3_dir>/Data/CRM3_p.dat<#>
"""
if ifile == '':
ifile = ephem_dir + 'Data/PE.EPH.gsme_in_Re_short'
#ifile = './PE.EPH.gsme_in_Re_short'
#
#--- set parameters (see crmflx for definitions)
#
ispeci = 1
iusesw = 1
iusemsh = 2
iusemsp = 0
smooth1 = 4
nflxget = 10
ndrophi = 2
ndroplo = 2
logflg = 2
rngtol = 4.0
fpchi = 80
fpclo = 20
fswimn = 0.0
fswi95 = 0.0
fswi50 = 0.0
fswisd = 0.0
#
#--- read solar wind database
#
xflux1, yflux1, zflux1, flxbin1, numbin1, numdat1 = cflx.swinit(ispeci)
#
#--- read magnetosheath databas
#
xflux2, yflux2, zflux2, flxbin2, numbin2, numdat2 = cflx.mshinit(ispeci)
#
#--- read magnetosphere database
#
xflux3, yflux3, zflux3, flxbin3, numbin3, numdat3,imapindx3,\
nsphvol3, ioffset3,joffset3,koffset3 = cflx.mspinit(ispeci)
#
#--- read the ephemeris data
#
data = mcf.read_data_file(ifile)
cdata = mcf.separate_data_to_arrays(data)
tlist = cdata[0]
xgsm = cdata[1]
ygsm = cdata[2]
zgsm = cdata[3]
for i in range(0, 28):
xkp = i / 3.0
line = ''
for j in range(0, len(tlist)):
idloc,fluxmn,flux95,flux50,fluxsd =\
cflx.crmflx(xkp,xgsm[j],ygsm[j],zgsm[j],ispeci,iusesw,\
fswimn,fswi95,fswi50,fswisd,iusemsh,iusemsp,smooth1,\
nflxget,ndrophi,ndroplo,logflg,rngtol,fpchi,fpclo,\
xflux1, yflux1, zflux1, flxbin1, numbin1, numdat1,\
xflux2, yflux2, zflux2, flxbin2, numbin2, numdat2,\
xflux3, yflux3, zflux3, flxbin3, numbin3, numdat3,\
nsphvol3, ioffset3,joffset3,koffset3,imapindx3)
line = line + '%13.1f\t' % tlist[j]
line = line + '%2d\t' % idloc
line = line + '%13.6e\t' % fluxmn
line = line + '%13.6e\t' % flux95
line = line + '%13.6e\t' % flux50
line = line + '%13.6e\n' % fluxsd
###ofile = crm3_dir +'Data/CRM3_p.dat' + tail[i]
ofile = './CRM_Out/CRM_p.dat' + tail[i]
with open(ofile, 'w') as fo:
fo.write(line)
def get_radzone_info():
"""
update rad_zone_info (a list of radiation zone entry/exit).
it will check this month and the next month.
"""
#
#--- find out today's date
#
today = time.strftime('%Y:%m', time.gmtime())
atemp = re.split(':', today)
year = int(float(atemp[0]))
month = int(float(atemp[1]))
#
#--- read the past radiation zone information
#
ifile = data_dir + 'rad_zone_info'
rdata = mcf.read_data_file(ifile)
#
#--- extract new radiation zone information of "ENTRY" and "EXIT"
#
entry_list = extract_radzone_info(year, month, 'RADENTRY')
exit_list = extract_radzone_info(year, month, 'RADEXIT')
#
#--- combined all three lists
#
clist = rdata + entry_list + exit_list
#
#--- to sort the list, first separate each entry to three elements
#
[cdat1, cdat2, cdat3] = mcf.separate_data_to_arrays(clist, '\t\t')
#
#--- the second elemet (cdat2) is chandra time; use that to sort the list in time order
#
cdat1 = numpy.array(cdat1)
cdat2 = numpy.array(cdat2)
cdat3 = numpy.array(cdat3)
idx = numpy.argsort(cdat2)
ent1 = cdat1[idx]
ent2 = cdat2[idx]
ent3 = cdat3[idx]
#
#--- recombine three element into one and make a time ordered list
#
l_line = ''
ind = ''
s_line = ''
for i in range(0, len(ent1)):
#
#--- check ENTRY/EXIT is in order
#
if ent1[i] != ind:
line = str(ent1[i]) + '\t\t' + str(ent2[i]) + '\t\t' + str(
ent3[i]) + '\n'
#
#--- remove duplicated lines
#
if line != l_line:
s_line = s_line + line
l_line = line
ind = ent1[i]
#
#--- move the old file
#
ofile = data_dir + 'rad_zone_info'
cmd = 'mv -f ' + ofile + ' ' + ofile + '~'
os.system(cmd)
#
#--- print out the data
#
with open(ofile, 'w') as fo:
fo.write(s_line)
