def useDataSeeker(startYear, startYdate, stopYear, stopYdate, extract, colList):
"extract data using dataseeker. Input: start, stop (e.g., 2012:03:13:22:41), the list name (e.g., mtahrc..hrcveto_avg), colnames: 'time,shevart_avg'"
#
#--- set dataseeker input file
#
(year1, month1, day1, hours1, minute1, second1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(startYear, startYdate)
(year2, month2, day2, hours2, minute2, second2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(stopYear, stopYdate)
f = open('./ds_file', 'w')
line = 'columns=' + extract + '\n'
f.write(line)
line = 'timestart=' + str(sectime1) + '\n'
f.write(line)
line = 'timestop=' + str(sectime2) + '\n'
f.write(line)
f.close()
cmd = 'punlearn dataseeker; dataseeker.pl infile=ds_file print=yes outfile=./ztemp.fits'
os.system(cmd)
cmd = 'dmlist "./ztemp.fits[cols '+ colList + '] " opt=data > ./zout_file'
os.system(cmd)
f = open('./zout_file', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./ds_file ./ztemp.fits ./zout_file')
return data
def useDataSeeker(startYear, startYdate, stopYear, stopYdate, extract, colList):
"extract data using dataseeker. Input: start, stop (e.g., 2012:03:13:22:41), the list name (e.g., mtahrc..hrcveto_avg), colnames: 'time,shevart_avg'"
#
#--- set dataseeker input file
#
(year1, month1, day1, hours1, minute1, second1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(startYear, startYdate)
(year2, month2, day2, hours2, minute2, second2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(stopYear, stopYdate)
f = open('./ds_file', 'w')
line = 'columns=' + extract + '\n'
f.write(line)
line = 'timestart=' + str(sectime1) + '\n'
f.write(line)
line = 'timestop=' + str(sectime2) + '\n'
f.write(line)
f.close()
cmd = 'punlearn dataseeker; dataseeker.pl infile=ds_file print=yes outfile=./ztemp.fits'
os.system(cmd)
cmd = 'dmlist "./ztemp.fits[cols '+ colList + '] " opt=data > ./zout_file'
os.system(cmd)
f = open('./zout_file', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./ds_file ./ztemp.fits ./zout_file')
return data
def convert_time(time, format = 0):
"""
convert time format from seconds from 1998.1.1 to dom or fractional year
Input: time --- a list of time in seconds
format --- if 0, convert into dom, otherwise, fractional year
Output: timeconverted --- a list of conveted time
"""
timeconverted = []
for ent in time:
stime = tcnv.convertCtimeToYdate(ent)
atime = tcnv.dateFormatConAll(stime)
if format == 0:
timeconverted.append(float(atime[7]))
else:
year = float(atime[0])
ydate = float(atime[6])
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
year += ydate /base
timeconverted.append(year)
return timeconverted
def convert_time(time, format = 0):
"""
convert time format from seconds from 1998.1.1 to dom or fractional year
Input: time --- a list of time in seconds
format --- if 0, convert into dom. if 1, ydate, otherwise, fractional year
Output: timeconverted --- a list of conveted time
"""
timeconverted = []
for ent in time:
stime = tcnv.convertCtimeToYdate(ent)
atime = tcnv.dateFormatConAll(stime)
if format == 0:
timeconverted.append(float(atime[7]))
elif format == 1:
ydate = float(atime[6])
timeconverted.append(ydate)
else:
year = float(atime[0])
ydate = float(atime[6])
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
year += ydate /base
timeconverted.append(year)
return timeconverted
def findTimeFromHead(file):
"""
isolate time part from a file and convert to DOM
Input: file --- fits file name
dom --- file creation time in DOM
"""
stime = extractTimePart(file)
ctime = tcnv.convertCtimeToYdate(stime)
(year, month, date, hours, minutes, seconds, ydate, dom, sectime) = tcnv.dateFormatConAll(ctime)
return int(dom)
def findTimeFromHead(file):
"""
isolate time part from a file and convert to DOM
Input: file --- fits file name
dom --- file creation time in DOM
"""
stime = extractTimePart(file)
ctime = tcnv.convertCtimeToYdate(stime)
(year, month, date, hours, minutes, seconds, ydate, dom,
sectime) = tcnv.dateFormatConAll(ctime)
return int(dom)
def convTimeFullColumn2(time_list):
"""
convert time format to fractional year for the entire array
Input: time_list --- a list of time
Output: converted --- a list of tine in dom
"""
converted = []
for ent in time_list:
time = tcnv.dateFormatConAll(ent)
year = time[0]
ydate = time[6]
dom = tcnv.YdateToDOM(year, ydate)
converted.append(dom)
return converted
def convTimeFullColumn(time_list):
"""
convert time format to fractional year for the entire array
Input: time_list --- a list of time
Output: converted --- a list of tine in fractional year
"""
converted = []
for ent in time_list:
time = tcnv.dateFormatConAll(ent)
year = time[0]
ydate = time[6]
chk = 4.0 * int(0.25 * year)
if year == chk:
base = 366
else:
base = 365
yf = year + ydate / base
converted.append(yf)
return converted
def extractGOESData(event, start, stop, comp_test='NA'):
"""
Extract GOES data from NOAA site, and create a locat data base.
input: event, interruption start/stop time (e.g., 20120313 2012:03:13:22:41 2012:03:14:13:57'
option comp_test is for testing (if comp_test == test, the test data will be read)
"""
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- reformat plotting start / stop time
#
(year3, month3, date3, hours3, minutes3, seconds3, ydate3, dom3, sectime3) = tcnv.dateFormatConAll(pYearStart, periodStart)
(year4, month4, date4, hours4, minutes4, seconds4, ydate4, dom4, sectime4) = tcnv.dateFormatConAll(pYearStop, periodStop)
#
#--- set input data type: old if pre 2011, otherwise new
#--- the first two is from http://goes.ngdc.noaa.gov/data/avg/ and the 'new' one is from http://www.swpc.noaa.gov/ftpdir/lists/pchan/
#--- although they all use p1, p2, and p5, the value are not compatible.
#
if year3 <= 2009:
dtype = 'G105'
elif year3 < 2011:
dtype = 'G115'
else:
dtype = 'new'
#
#--- create a list of html address from which we extract GOES data
#
htmlList = []
#
#--- for the starting year and the ending year is same
#
if year3 == year4:
#
#--- for the case the starting month and ending month is same
#
if month3 == month4:
tmon = str(month3)
if month3 < 10:
tmon = '0' + tmon
for tday in range(date3, date4+1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year3) + tmon + tday
#
#--- after 2012, the data are obtained from different site. data are largely missing
#--- 2011, and previous to that we have all record at ngdc site
#
if dtype == 'new':
# html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
html = 'http://services.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
else:
#
#---- for the case, the period goes over two months
#
if month3 == 2:
chk = 4.0 * int(0.25 * year3)
if chk == year3:
endDate = 29
else:
endDate = 28
elif month3 == 1 or month3 == 3 or month3 == 5 or month3 == 7 or month3 == 8 or month3 == 10:
endDate = 31
else:
endDate = 30
tmon = str(month3)
if month3 < 10:
tmon = '0' + tmon
for tday in range(date3, endDate+1):
timeStamp = str(year3) + tmon + str(tday)
if dtype == 'new':
html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
tmon = str(month4)
if month4 < 10:
tmon = '0' + tmon
for tday in range(1, date4+1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year3) + tmon + tday
if dtype == 'new':
html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
else:
#
#--- for the case the period goes over two years
#
for tday in range(date3, 32):
timeStamp = str(year3) + tmon + str(tday)
if dtype == 'new':
html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
for tday in range(1, date4+1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year4) + tmon + tday
if dtype == 'new':
html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year4) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
#
#--- prepare to print out data
#
if comp_test == 'test':
ofile = test_data_dir + event + '_goes.txt'
else:
ofile = data_dir + event + '_goes.txt'
out = open(ofile, 'w')
line = 'Science Run Interruption: ' + str(start) +'\n\n'
out.write(line)
out.write('dofy\t\tp1\t\t\tp2\t\t\tp5\n')
out.write("-------------------------------------------------------------------\n")
#
#--- now extract data from NOAA web site
#
for html in htmlList:
if comp_test == 'test':
atemp = re.split('pchan\/', html)
afile = house_keeping + 'NOAO_data/' + atemp[1]
cmd = 'cp ' + afile + ' ./Working_dir/temp_data'
else:
cmd = 'lynx -source ' + html + '>./Working_dir/temp_data'
os.system(cmd)
f = open('./Working_dir/temp_data', 'r')
data = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./Working_dir/temp_data')
#
#--- extract needed data and print them out in a data file
#
#
#--- if data are collected after 2011, start here
#
if dtype == 'new':
for ent in data:
atemp = re.split('\s+|\t+', ent)
if ent and atemp[0].isdigit():
timestamp = atemp[0] + ':' + atemp[1] + ':' + atemp[2] + ':' + atemp[3][0] + atemp[3][1] + ':' + atemp[3][2] + atemp[3][3] + ':00'
(dyear, dmonth, dday, dhours, dminutes, dseconds, dydate) = tcnv.dateFormatCon(timestamp)
if dyear == pYearStart:
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' % (dydate, float(atemp[6]), float(atemp[7]), float(atemp[10]))
out.write(line)
else:
#
#--- for the case, the period goes over two years
#
chk = 4.0 * int(0.25 * pYearStart)
if chk == pYearStart:
base = 366
else:
base = 365
dydate += base
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' % (dydate, float(atemp[6]), float(atemp[7]), float(atemp[10]))
out.write(line)
#
#--- if the data is collected before 2011, start here
#
else:
for ent in data:
atemp = re.split('\s+|\t+', ent)
if ent and atemp[0].isdigit():
dyear = atemp[0][0] + atemp[0][1]
dyear = int(dyear)
if dyear > 90:
dyear += 1900
else:
dyear += 2000
dydate = float(atemp[2])
dhours = float(atemp[1][0] + atemp[1][1])
dminutes = float(atemp[1][2] + atemp[1][3])
dydate += (dhours/24.0 + dminutes / 1440.0)
if dyear == pYearStart:
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' % (dydate, float(atemp[10]), float(atemp[11]), float(atemp[14]))
out.write(line)
else:
#
#--- for the case, the period goes over two years
#
chk = 4.0 * int(0.25 * pYearStart)
if chk == pYearStart:
base = 366
else:
base = 365
dydate += base
dydate += base
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' % (dydate, float(atemp[10]), float(atemp[11]), float(atemp[14]))
out.write(line)
if len(htmlList) > 0:
out.close()
def clean_cti_data_table(dir):
"""
remmove data points which are extrme outlyers and then clean up output data tables.
Input: dir --- the directory where the data files are kept
Output: updated data files in the directory <dir>
"""
dropped = data_dir + dir + '/dropped_data'
fo = open(dropped, 'w')
dropped_obsids = []
for elm in elm_list:
line = 'ELM: ' + elm + '\n'
fo.write(line)
for ccd in range(0, 10):
if ccd == 5 or ccd == 7:
drop_factor = 5.0 #--- drop_factor sets the boundray of the outlyer: how may signam away?
else:
drop_factor = 4.0
#
#--- check the input file exists
#
dname = data_dir + dir + '/' + elm + '_ccd' + str(ccd)
chk = mcf.isFileEmpty(dname)
if chk > 0:
line = 'CCD: ' + str(ccd) + '\n'
fo.write(line)
f = open(dname, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- separate data into separate array data sets
#
dcolumns = separate_data(data)
cti = ['' for x in range(4)]
cti[0] = dcolumns[0]
cti[1] = dcolumns[1]
cti[2] = dcolumns[2]
cti[3] = dcolumns[3]
obsid = dcolumns[10]
dom = []
for ent in dcolumns[8]:
time_list = tcnv.dateFormatConAll(ent)
dom.append(time_list[7])
#
#--- go around quads
#
drop_list = []
for i in range(0, 4):
line = "QUAD" + str(i)+ '\n'
fo.write(line)
#
#--- fit a lienar line
#
(intc, slope) = linear_fit(dom, cti[i])
sum = 0
#
#--- compute a deviation from the fitted line
#
diff_save = []
for j in range(0, len(dom)):
diff = float(cti[i][j]) - (intc + slope * float(dom[j]))
diff_save.append(diff)
sum += diff * diff
sigma = math.sqrt(sum/len(dom))
#
#--- find outlyers
#
out_val = drop_factor * sigma
for j in range(0, len(dom)):
if diff_save[j] > out_val:
drop_list.append(j)
fo.write(data[j])
fo.write('\n')
#
#--- clean up the list; removing duplicated lines
#
drop_list = mcf.removeDuplicate(drop_list, chk = 0)
cleaned_data = []
for i in range(0, len(dom)):
chk = 0
for comp in drop_list:
if i == comp:
chk = 1
break
if chk == 0:
cleaned_data.append(data[i])
cleaned_data = mcf.removeDuplicate(cleaned_data, chk = 0)
for ent in drop_list:
dropped_obsids.append(obsid[ent])
f = open(dname, 'w')
for ent in cleaned_data:
f.write(ent)
f.write('\n')
f.close()
fo.close()
dropped_obsids = mcf.removeDuplicate(dropped_obsids, chk = 0)
out = data_dir + dir + '/bad_data_obsid'
f = open(out, 'w')
for ent in dropped_obsids:
f.write(ent)
f.write('\n')
def plotEphinMain(event, start, stop, comp_test = 'NA'):
'read Ephin data from data_dir and plot them. Input: event, interruption start and stop time (e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57)'
#
#--- read radiation zone information
#
radZone = ptrf.readRadZone(event)
#
#--- read EPHIN data
#
if comp_test == 'test':
file = test_data_dir + event + '_eph.txt'
plot_out = test_ephin_dir
else:
file = data_dir + event + '_eph.txt'
plot_out = ephin_dir
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dofy = []
prtn1 = []
prtn2 = []
prtn3 = []
dcnt = 0
dataset= 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
btemp = re.split('\.', atemp[0])
if ent and btemp[0].isdigit():
dofy.append(atemp[0])
val1 = float(atemp[1])
if val1 <= 0:
val1 = 1e-5
val2 = float(atemp[2])
if val2 <= 0:
val2 = 1e-5
val3 = float(atemp[3])
if val3 <= 0:
val3 = 1e-5
prtn1.append(math.log10(val1))
prtn2.append(math.log10(val2))
prtn3.append(math.log10(val3))
else:
#
#--- checking which data set, old one: p4, p41, e1300. new one: hrc, e150, e1300
#
m = re.search('hrc', ent)
if m is not None:
dataset = 1
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- if the interuption go over two years, adjust the ending ydate to that of the previous year
#
if year2 > year1:
chk = 4.0 * int(0.25 * year1)
if chk == year1:
base = 366
else:
base = 365
ydate2 += base
#
#--- plot data
#
if pannelNum == 1:
if year1 < 2014:
plotEphin(dofy, prtn1, prtn2, prtn3, ydate1, ydate2, plotStart, plotStop, radZone, dataset)
else:
plotEphin(dofy, prtn1, 'NA', 'NA', ydate1, ydate2, plotStart, plotStop, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
#
#--- if the interruption period cannot be covered by one plotting panel, create as many panels as we need to cover the period.
#
else:
pstart = plotStart
prange = pannelNum + 1
if year1 < 2014:
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotEphin(dofy, prtn1, prtn2, prtn3, ydate1, 'NA', pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
elif i == pannelNum:
plotEphin(dofy, prtn1, prtn2, prtn3, 'NA', ydate2, pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt'+ str(i) + '.png'
os.system(cmd)
else:
plotEphin(dofy, prtn1, prtn2, prtn3, 'NA', 'NA', pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt'+ str(i) + '.png'
os.system(cmd)
pstart = pend
else:
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotEphin(dofy, prtn1, 'NA', 'NA', ydate1, 'NA', pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
elif i == pannelNum:
plotEphin(dofy, prtn1, 'NA', 'NA', 'NA', ydate2, pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt'+ str(i) + '.png'
os.system(cmd)
else:
plotEphin(dofy, prtn1, 'NA', 'NA', 'NA', 'NA', pstart, pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt'+ str(i) + '.png'
os.system(cmd)
pstart = pend
#
#--- plot intro page
#
pend = plotStart + 5
if year1 < 2014:
plotIntro(dofy, prtn2, ydate1, ydate2, plotStart, pend, radZone, dataset, year1)
else:
plotIntro(dofy, prtn1, ydate1, ydate2, plotStart, pend, radZone, dataset, year1)
cmd = 'mv ./intro_out.png ' + intro_dir + event + '_intro.png'
os.system(cmd)
def plotGOESMain(event, start, stop, comp_test='NA'):
'read GOES data from data_dir and plot them. Input: event, interruption start and stop time (e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57)'
#
#--- read radiation zone information
#
radZone = ptrf.readRadZone(event)
#
#--- read GOES data
#
if comp_test == 'test':
file = test_data_dir + event + '_goes.txt'
plot_out = test_goes_dir
else:
file = data_dir + event + '_goes.txt'
plot_out = goes_dir
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dofy = []
p1 = []
p2 = []
p5 = []
dcnt = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
btemp = re.split('\.', atemp[0])
if ent and btemp[0].isdigit():
dofy.append(atemp[0])
val1 = float(atemp[1])
if val1 <= 0:
val1 = 1e-5
val2 = float(atemp[2])
if val2 <= 0:
val2 = 1e-5
val3 = float(atemp[3])
if val3 <= 0:
val3 = 1e-5
p1.append(math.log10(val1))
p2.append(math.log10(val2))
p5.append(math.log10(val3))
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1,
sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2,
sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- if the interuption go over two years, adjust the ending ydate to that of the previous year
#
if year2 > year1:
chk = 4.0 * int(0.25 * year1)
if chk == year1:
base = 366
else:
base = 365
ydate2 += base
#
#--- plot data
#
if pannelNum == 1:
plotGOES(dofy, p1, p2, p5, ydate1, ydate2, plotStart, plotStop,
radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes.png'
os.system(cmd)
#
#--- if the interruption period cannot be covered by one plotting panel, create as many panels as we need to cover the period.
#
else:
pstart = plotStart
prange = pannelNum + 1
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotGOES(dofy, p1, p2, p5, ydate1, 'NA', pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes.png'
os.system(cmd)
elif i == pannelNum:
plotGOES(dofy, p1, p2, p5, 'NA', ydate2, pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes_pt' + str(
i) + '.png'
os.system(cmd)
else:
plotGOES(dofy, p1, p2, p5, 'NA', 'NA', pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes_pt' + str(
i) + '.png'
os.system(cmd)
pstart = pend
def ephinDataExtract(event, start, stop, comp_test='NA'):
"extract EPIN related quantities and creates a data table for given event, start and stop time (format: 2012:03:13:22:41)"
begin = start + ':00' #---- to use dateFormatCon correctly, need to add "sec" part
end = stop + ':00'
#
#--- convert time format
#
(year1, month1, day1, hours1, minutes1, seconds1,
ydate1) = tcnv.dateFormatCon(begin)
(year2, month2, day2, hours2, minutes2, seconds2,
ydate2) = tcnv.dateFormatCon(end)
#
#--- change time format and find data collecting period (starts 2 days before the interruption and ends at least 5 days after the stating)
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- read ephin data using arc4gl
#
ephinList = itrf.useArc4gl(
'retrieve',
'flight',
'ephin',
1,
'ephrates',
pYearStart,
periodStart,
pYearStop,
periodStop,
deposit='./Working_dir',
)
#
#--- extract needed data
#
xdate = []
p4 = []
p41 = []
e150 = []
e1300 = []
ecnt = 0
for fits in ephinList:
#
#--- use dmlist
#
if pYearStart < 2011:
line = fits + '[cols TIME, SCP4, SCP41, SCE1300]'
else:
line = fits + '[cols TIME, SCE150, SCE1300]'
cmd = 'dmlist "' + line + '" opt="data" > ./zout'
os.system(cmd)
f = open('./zout', 'r')
input = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./zout')
for ent in input:
ent.lstrip()
atemp = re.split('\s+|\t+', ent)
if atemp[0].isdigit():
line = tcnv.convertCtimeToYdate(float(atemp[1]))
(year, month, day, hours, minute, second, ydate, dom,
sectime) = tcnv.dateFormatConAll(line)
#
#--- checking the case the year change
#
if ecnt > 0 and year != startYear:
ydate += base
elif ecnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
xdate.append(ydate)
if pYearStart < 2011:
if float(atemp[3]) == 0:
atemp[3] = 1.0e-4
if float(atemp[4]) == 0:
atemp[4] = 1.0e-4
p4.append(atemp[2])
p41.append(atemp[3])
e1300.append(atemp[4])
else:
if float(atemp[2]) == 0:
atemp[2] = 1.0e-4
if float(atemp[3]) == 0:
atemp[3] = 1.0e-4
e150.append(atemp[2])
e1300.append(atemp[3])
ecnt += 1
os.system('rm ./Working_dir/*fits')
#
#--- using DataSeeker, extread HRC sheild rate (only if year > 2011)
#
if pYearStart >= 2011:
hrcData = itrf.useDataSeeker(pYearStart, periodStart, pYearStop,
periodStop, 'mtahrc..hrcveto_avg',
'time,shevart_avg')
time = []
veto = []
hcnt = 0
#
#--- there are slightly different input line format; so we need to different ways to read the line
#
for ent in hrcData:
m1 = re.search('----', ent)
m2 = re.search('ROW', ent)
m3 = re.search('DATASEEK', ent)
if ent and m1 == None and m2 == None and m3 == None:
atemp = re.split('\s+|\t+', ent)
if len(atemp) == 4:
ttime = tcnv.convertCtimeToYdate(float(atemp[2]))
temp = re.split(':', ttime)
year = int(temp[0])
dofy = float(temp[1]) + float(temp[2]) / 24 + float(
temp[3]) / 1440 + float(temp[4]) / 86400
#
#--- checking the case the year changes
#
if hcnt > 0 and year != startYear:
dofy += base
elif hcnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
time.append(dofy)
veto.append(atemp[3])
hcnt += 1
else:
ttime = tcnv.convertCtimeToYdate(float(atemp[1]))
temp = re.split(':', ttime)
year = int(temp[0])
dofy = float(temp[1]) + float(temp[2]) / 24 + float(
temp[3]) / 1440 + float(temp[4]) / 86400
if hcnt > 0 and year != startYear:
dofy += base
elif hcnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
time.append(dofy)
veto.append(atemp[2])
hcnt += 1
#
#--- matching timing between electron data and hrc data
#
hrc = len(e150) * [0]
j = 0
k = 0
#
#--- find the begining
#
if time[0] < xdate[0]:
while time[j] < xdate[0]:
j += 1
if j >= hcnt:
print "Time span does not overlap. Abort the process."
exit(1)
elif time[0] > xdate[0]:
while time[0] > xdate[k]:
k += 1
if k >= ecnt:
print "Time span does not overlap. Abort the process."
exit(1)
hrc[k] = veto[j]
tspace = 1.38888888888e-3 / base #--- setting timing bin size: base is given in hrc loop
for i in range(k + 1, ecnt):
tbeg = xdate[i] - tspace
tend = xdate[i] + tspace
if j > hcnt - 2:
hrc[i] = veto[
hcnt -
1] #---- if the hrc data runs out, just repeat the last data point value
elif time[j] >= tbeg and time[j] <= tend:
hrc[i] = veto[j]
elif time[j] < tbeg:
while time[j] < tbeg:
j += 1
hrc[i] = veto[j]
elif time[j] > tend:
while time[j] > tend:
j -= 1
hrc[i] = veto[j]
#
#--- print out data
#
if comp_test == 'test':
file = test_data_dir + event + '_eph.txt'
else:
file = data_dir + event + '_eph.txt'
f = open(file, 'w')
line = 'Science Run Interruption: ' + start + '\n\n'
f.write(line)
if pYearStart < 2011:
f.write('dofy\t\tp4\t\t\tp41\t\t\te1300\n')
f.write(
'-------------------------------------------------------------------\n'
)
for m in range(0, ecnt):
line = '%4.3f\t\t%4.3e\t%4.3e\t%4.3e\n' % (float(
xdate[m]), float(p4[m]), float(p41[m]), float(e1300[m]))
f.write(line)
else:
f.write('dofy\t\thrc\t\te150\t\te1300\n')
f.write(
'-------------------------------------------------------------------\n'
)
for m in range(0, ecnt):
line = '%4.3f\t\t%4.3e\t%4.3e\t%4.3e\n' % (float(
xdate[m]), float(hrc[m]), float(e150[m]), float(e1300[m]))
f.write(line)
f.close()
def ephinDataExtract(event, start, stop, comp_test ='NA'):
"extract EPIN related quantities and creates a data table for given event, start and stop time (format: 2012:03:13:22:41)"
begin = start + ':00' #---- to use dateFormatCon correctly, need to add "sec" part
end = stop + ':00'
#
#--- convert time format
#
(year1, month1, day1, hours1, minutes1, seconds1, ydate1) = tcnv.dateFormatCon(begin)
(year2, month2, day2, hours2, minutes2, seconds2, ydate2) = tcnv.dateFormatCon(end)
#
#--- change time format and find data collecting period (starts 2 days before the interruption and ends at least 5 days after the stating)
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- read ephin data using arc4gl
#
ephinList = itrf.useArc4gl('retrieve', 'flight', 'ephin', 1, 'ephrates', pYearStart, periodStart, pYearStop, periodStop, deposit='./Working_dir',)
#
#--- extract needed data
#
xdate = []
p4 = []
p41 = []
e150 = []
e1300 = []
ecnt = 0
for fits in ephinList:
#
#--- use dmlist
#
if pYearStart < 2011:
line = fits + '[cols TIME, SCP4, SCP41, SCE1300]'
else:
line = fits + '[cols TIME, SCE150, SCE1300]'
cmd = 'dmlist "' + line + '" opt="data" > ./zout'
os.system(cmd)
f = open('./zout', 'r')
input = [line.strip() for line in f.readlines()]
f.close()
os.system('rm ./zout')
for ent in input:
ent.lstrip()
atemp = re.split('\s+|\t+', ent)
if atemp[0].isdigit():
line = tcnv.convertCtimeToYdate(float(atemp[1]))
(year, month, day, hours, minute, second, ydate, dom, sectime) = tcnv.dateFormatConAll(line)
#
#--- checking the case the year change
#
if ecnt > 0 and year != startYear:
ydate += base
elif ecnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
xdate.append(ydate)
if pYearStart < 2011:
if float(atemp[3]) == 0:
atemp[3] = 1.0e-4
if float(atemp[4]) == 0:
atemp[4] = 1.0e-4
p4.append(atemp[2])
p41.append(atemp[3])
e1300.append(atemp[4])
else:
if float(atemp[2]) == 0:
atemp[2] = 1.0e-4
if float(atemp[3]) == 0:
atemp[3] = 1.0e-4
e150.append(atemp[2])
e1300.append(atemp[3])
ecnt += 1
os.system('rm ./Working_dir/*fits')
#
#--- using DataSeeker, extread HRC sheild rate (only if year > 2011)
#
if pYearStart >= 2011:
hrcData = itrf.useDataSeeker(pYearStart, periodStart, pYearStop, periodStop, 'mtahrc..hrcveto_avg', 'time,shevart_avg')
time = []
veto = []
hcnt = 0
#
#--- there are slightly different input line format; so we need to different ways to read the line
#
for ent in hrcData:
m1 = re.search('----', ent)
m2 = re.search('ROW', ent)
m3 = re.search('DATASEEK', ent)
if ent and m1 == None and m2 == None and m3 == None:
atemp = re.split('\s+|\t+', ent)
if len(atemp) == 4:
ttime = tcnv.convertCtimeToYdate(float(atemp[2]))
temp = re.split(':', ttime)
year = int(temp[0])
dofy = float(temp[1]) + float(temp[2]) / 24 + float(temp[3]) / 1440 + float(temp[4]) / 86400
#
#--- checking the case the year changes
#
if hcnt > 0 and year != startYear:
dofy += base
elif hcnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
time.append(dofy)
veto.append(atemp[3])
hcnt += 1
else:
ttime = tcnv.convertCtimeToYdate(float(atemp[1]))
temp = re.split(':', ttime)
year = int(temp[0])
dofy = float(temp[1]) + float(temp[2]) / 24 + float(temp[3]) / 1440 + float(temp[4]) / 86400
if hcnt > 0 and year != startYear:
dofy += base
elif hcnt == 0:
startYear = year
chk = 4.0 * int(0.25 * year)
if chk == year:
base = 366
else:
base = 365
time.append(dofy)
veto.append(atemp[2])
hcnt += 1
#
#--- matching timing between electron data and hrc data
#
hrc = len(e150) * [0]
j = 0
k = 0
#
#--- find the begining
#
if time[0] < xdate[0]:
while time[j] < xdate[0]:
j += 1
if j >= hcnt:
print "Time span does not overlap. Abort the process."
exit(1)
elif time[0] > xdate[0]:
while time[0] > xdate[k]:
k += 1
if k >= ecnt:
print "Time span does not overlap. Abort the process."
exit(1)
hrc[k] = veto[j]
tspace = 1.38888888888e-3 / base #--- setting timing bin size: base is given in hrc loop
for i in range(k+1, ecnt):
tbeg = xdate[i] - tspace
tend = xdate[i] + tspace
if j > hcnt - 2:
hrc[i] = veto[hcnt -1] #---- if the hrc data runs out, just repeat the last data point value
elif time[j] >= tbeg and time[j] <= tend:
hrc[i] = veto[j]
elif time[j] < tbeg:
while time[j] < tbeg:
j += 1
hrc[i] = veto[j]
elif time[j] > tend:
while time[j] > tend:
j -= 1
hrc[i] = veto[j]
#
#--- print out data
#
if comp_test == 'test':
file = test_data_dir + event + '_eph.txt'
else:
file = data_dir + event + '_eph.txt'
f = open(file, 'w')
line = 'Science Run Interruption: ' + start + '\n\n'
f.write(line)
if pYearStart < 2011:
f.write('dofy\t\tp4\t\t\tp41\t\t\te1300\n')
f.write('-------------------------------------------------------------------\n')
for m in range(0, ecnt):
line = '%4.3f\t\t%4.3e\t%4.3e\t%4.3e\n' % (float(xdate[m]), float(p4[m]), float(p41[m]), float(e1300[m]))
f.write(line)
else:
f.write('dofy\t\thrc\t\te150\t\te1300\n')
f.write('-------------------------------------------------------------------\n')
for m in range(0, ecnt):
line = '%4.3f\t\t%4.3e\t%4.3e\t%4.3e\n' % (float(xdate[m]), float(hrc[m]), float(e150[m]), float(e1300[m]))
f.write(line)
f.close()
def extractGOESData(event, start, stop, comp_test='NA'):
"""
Extract GOES data from NOAA site, and create a locat data base.
input: event --- event name
start --- starting time
stop --- stopping time
(e.g., 20120313 2012:03:13:22:41 2012:03:14:13:57'
comp_test --- option; if given, testing
(if comp_test == test, the test data will be read)
output: <data_dir>/<event>_goes.txt
<stat_dir>/<event>_geos_stat
"""
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1,
sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2,
sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- reformat plotting start / stop time
#
(year3, month3, date3, hours3, minutes3, seconds3, ydate3, dom3,
sectime3) = tcnv.dateFormatConAll(pYearStart, periodStart)
(year4, month4, date4, hours4, minutes4, seconds4, ydate4, dom4,
sectime4) = tcnv.dateFormatConAll(pYearStop, periodStop)
#
#--- set input data type: old if pre 2011, otherwise new
#--- the first two is from http://goes.ngdc.noaa.gov/data/avg/ and the 'new' one is from http://www.swpc.noaa.gov/ftpdir/lists/pchan/
#--- although they all use p1, p2, and p5, the value are not compatible.
#
if year3 <= 2009:
dtype = 'G105'
elif year3 < 2011:
dtype = 'G115'
else:
dtype = 'new'
#
#--- create a list of html address from which we extract GOES data
#
htmlList = []
#
#--- for the starting year and the ending year is same
#
if year3 == year4:
#
#--- for the case the starting month and ending month is same
#
if month3 == month4:
tmon = str(month3)
if month3 < 10:
tmon = '0' + tmon
for tday in range(date3, date4 + 1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year3) + tmon + tday
#
#--- after 2012, the data are obtained from different site. data are largely missing
#--- 2011, and previous to that we have all record at ngdc site
#
if dtype == 'new':
# html = 'http://www.swpc.noaa.gov/ftpdir/lists/pchan/' + timeStamp + '_Gp_pchan_5m.txt'
html = '/data/mta4/www/DAILY/mta_rad/GOES/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(
year3
) + '/' + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
else:
#
#---- for the case, the period goes over two months
#
if month3 == 2:
chk = 4.0 * int(0.25 * year3)
if chk == year3:
endDate = 29
else:
endDate = 28
elif month3 == 1 or month3 == 3 or month3 == 5 or month3 == 7 or month3 == 8 or month3 == 10:
endDate = 31
else:
endDate = 30
tmon = str(month3)
if month3 < 10:
tmon = '0' + tmon
for tday in range(date3, endDate + 1):
timeStamp = str(year3) + tmon + str(tday)
if dtype == 'new':
html = '/data/mta4/www/DAILY/mta_rad/GOES/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3)
html = html + '/' + dtype + syear[2] + syear[
3] + tmon + '.TXT'
htmlList.append(html)
tmon = str(month4)
if month4 < 10:
tmon = '0' + tmon
for tday in range(1, date4 + 1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year3) + tmon + tday
if dtype == 'new':
html = '/data/mta4/www/DAILY/mta_rad/GOES/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(
year3) + '/'
html = html + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
else:
#
#--- for the case the period goes over two years
#
for tday in range(date3, 32):
timeStamp = str(year3) + tmon + str(tday)
if dtype == 'new':
html = '/data/mta4/www/DAILY/mta_rad/GOES/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year3) + '/'
html = html + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
for tday in range(1, date4 + 1):
if tday < 10:
tday = '0' + str(tday)
else:
tday = str(tday)
timeStamp = str(year4) + tmon + tday
if dtype == 'new':
html = '/data/mta4/www/DAILY/mta_rad/GOES/' + timeStamp + '_Gp_pchan_5m.txt'
else:
syear = str(year3)
html = 'http://goes.ngdc.noaa.gov/data/avg/' + str(year4) + '/'
html = html + dtype + syear[2] + syear[3] + tmon + '.TXT'
htmlList.append(html)
#
#--- prepare to print out data
#
if comp_test == 'test':
ofile = test_data_dir + event + '_goes.txt'
else:
ofile = data_dir + event + '_goes.txt'
out = open(ofile, 'w')
line = 'Science Run Interruption: ' + str(start) + '\n\n'
out.write(line)
out.write('dofy\t\tp1\t\t\tp2\t\t\tp5\n')
out.write(
"-------------------------------------------------------------------\n"
)
#
#--- now extract data from NOAA web site
#
for html in htmlList:
f = open(html, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
#
#--- extract needed data and print them out in a data file
#
#
#--- if data are collected after 2011, start here
#
if dtype == 'new':
for ent in data:
atemp = re.split('\s+|\t+', ent)
if ent and atemp[0].isdigit():
timestamp = atemp[0] + ':' + atemp[1] + ':' + atemp[
2] + ':' + atemp[3][0]
timestamp = timestamp + atemp[3][1] + ':' + atemp[3][
2] + atemp[3][3] + ':00'
(dyear, dmonth, dday, dhours, dminutes, dseconds,
dydate) = tcnv.dateFormatCon(timestamp)
if dyear == pYearStart:
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' %\
(dydate, float(atemp[6]), float(atemp[7]), float(atemp[10]))
out.write(line)
else:
#
#--- for the case, the period goes over two years
#
chk = 4.0 * int(0.25 * pYearStart)
if chk == pYearStart:
base = 366
else:
base = 365
dydate += base
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' %\
(dydate, float(atemp[6]), float(atemp[7]), float(atemp[10]))
out.write(line)
#
#--- if the data is collected before 2011, start here
#
else:
for ent in data:
atemp = re.split('\s+|\t+', ent)
if ent and atemp[0].isdigit():
dyear = atemp[0][0] + atemp[0][1]
dyear = int(dyear)
if dyear > 90:
dyear += 1900
else:
dyear += 2000
dydate = float(atemp[2])
dhours = float(atemp[1][0] + atemp[1][1])
dminutes = float(atemp[1][2] + atemp[1][3])
dydate += (dhours / 24.0 + dminutes / 1440.0)
if dyear == pYearStart:
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' %\
(dydate, float(atemp[10]), float(atemp[11]), float(atemp[14]))
out.write(line)
else:
#
#--- for the case, the period goes over two years
#
chk = 4.0 * int(0.25 * pYearStart)
if chk == pYearStart:
base = 366
else:
base = 365
dydate += base
dydate += base
if dydate >= plotStart and dydate <= plotStop:
line = '%4.3f\t\t%3.2e\t%3.2e\t%3.2e\n' %\
(dydate, float(atemp[10]), float(atemp[11]), float(atemp[14]))
out.write(line)
if len(htmlList) > 0:
out.close()
def plotEphinMain(event, start, stop, comp_test='NA'):
'read Ephin data from data_dir and plot them. Input: event, interruption start and stop time (e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57)'
#
#--- read radiation zone information
#
radZone = ptrf.readRadZone(event)
#
#--- read EPHIN data
#
if comp_test == 'test':
file = test_data_dir + event + '_eph.txt'
plot_out = test_ephin_dir
else:
file = data_dir + event + '_eph.txt'
plot_out = ephin_dir
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dofy = []
prtn1 = []
prtn2 = []
prtn3 = []
dcnt = 0
dataset = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
btemp = re.split('\.', atemp[0])
if ent and btemp[0].isdigit():
dofy.append(atemp[0])
val1 = float(atemp[1])
if val1 <= 0:
val1 = 1e-5
val2 = float(atemp[2])
if val2 <= 0:
val2 = 1e-5
val3 = float(atemp[3])
if val3 <= 0:
val3 = 1e-5
prtn1.append(math.log10(val1))
prtn2.append(math.log10(val2))
prtn3.append(math.log10(val3))
else:
#
#--- checking which data set, old one: p4, p41, e1300. new one: hrc, e150, e1300
#
m = re.search('hrc', ent)
if m is not None:
dataset = 1
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1,
sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2,
sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- if the interuption go over two years, adjust the ending ydate to that of the previous year
#
if year2 > year1:
chk = 4.0 * int(0.25 * year1)
if chk == year1:
base = 366
else:
base = 365
ydate2 += base
#
#--- plot data
#
if pannelNum == 1:
if year1 < 2014:
plotEphin(dofy, prtn1, prtn2, prtn3, ydate1, ydate2, plotStart,
plotStop, radZone, dataset)
else:
plotEphin(dofy, prtn1, 'NA', 'NA', ydate1, ydate2, plotStart,
plotStop, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
#
#--- if the interruption period cannot be covered by one plotting panel, create as many panels as we need to cover the period.
#
else:
pstart = plotStart
prange = pannelNum + 1
if year1 < 2014:
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotEphin(dofy, prtn1, prtn2, prtn3, ydate1, 'NA', pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
elif i == pannelNum:
plotEphin(dofy, prtn1, prtn2, prtn3, 'NA', ydate2, pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt' + str(
i) + '.png'
os.system(cmd)
else:
plotEphin(dofy, prtn1, prtn2, prtn3, 'NA', 'NA', pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt' + str(
i) + '.png'
os.system(cmd)
pstart = pend
else:
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotEphin(dofy, prtn1, 'NA', 'NA', ydate1, 'NA', pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph.png'
os.system(cmd)
elif i == pannelNum:
plotEphin(dofy, prtn1, 'NA', 'NA', 'NA', ydate2, pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt' + str(
i) + '.png'
os.system(cmd)
else:
plotEphin(dofy, prtn1, 'NA', 'NA', 'NA', 'NA', pstart,
pend, radZone, dataset)
cmd = 'mv ./out.png ' + plot_out + event + '_eph_pt' + str(
i) + '.png'
os.system(cmd)
pstart = pend
#
#--- plot intro page
#
pend = plotStart + 5
if year1 < 2014:
plotIntro(dofy, prtn2, ydate1, ydate2, plotStart, pend, radZone,
dataset, year1)
else:
plotIntro(dofy, prtn1, ydate1, ydate2, plotStart, pend, radZone,
dataset, year1)
cmd = 'mv ./intro_out.png ' + intro_dir + event + '_intro.png'
os.system(cmd)
def printEachHtml(event, start, stop, gap, stopType):
'create indivisual event html page. input event name, interruption start/stop time, gap, and type (auto/manual): example: 20031202 2003:12:02:17:31 2003:12:04:14:27 139.8 auto'
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- check whether we need multiple pannels
#
pannelNum = int((plotStop - plotStart) / 5)
#
#--- choose a template
#
atemp = re.split(':', start)
year = int(atemp[0])
if year < 2011:
file = house_keeping + 'sub_html_template'
elif year < 2014:
file = house_keeping + 'sub_html_template_2011'
else:
file = house_keeping + 'sub_html_template_2014'
#
#--- read the template and start substituting
#
data = open(file).read()
data = re.sub('#header_title#', event, data)
data = re.sub('#main_title#', event, data)
data = re.sub('#sci_run_stop#', start, data)
data = re.sub('#sci_run_start#', stop, data)
data = re.sub('#interruption#', gap, data)
data = re.sub('#trigger#', stopType, data)
noteN = event + '.txt'
data = re.sub('#note_name#', noteN, data)
#
#--- ACA (NOAA) radiation data
#
aceData = event + '_dat.txt'
data = re.sub('#ace_data#', aceData, data)
file = stat_dir + event + '_ace_stat'
stat = open(file).read()
data = re.sub('#ace_table#', stat, data)
line = event + '.png"'
for i in range(2, pannelNum+1):
padd = ' alt="main plot" style="width:100%">\n<br />\n<img src = "../Main_plot/' + event + '_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#ace_plot#', line , data)
#
#---EPHIN data
#
ephData = event + '_eph.txt'
data = re.sub('#eph_data#', ephData, data)
file = stat_dir + event + '_ephin_stat'
stat = open(file).read()
data = re.sub('#eph_table#', stat, data)
line = event + '_eph.png"'
for i in range(2, pannelNum+1):
padd = ' alt="eph plot" style="width:100%">\n<br />\n<img src = "../Ephin_plot/' + event + '_eph_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#eph_plot#', line , data)
#
#---GOES data
#
goesData = event + '_goes.txt'
data = re.sub('#goes_data#', goesData, data)
file = stat_dir + event + '_goes_stat'
stat = open(file).read()
data = re.sub('#goes_table#', stat, data)
line = event + '_goes.png"'
for i in range(2, pannelNum+1):
padd = ' alt="goes plot" style="width:100%"> \n<br />\n<img src = "../GOES_plot/' + event + '_goes_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#goes_plot#', line , data)
if year1 >= 2011:
data = re.sub('GOES-11', 'GOES-15', data)
file = web_dir + 'Html_dir/' + event + '.html'
f = open(file, 'w')
f.write(data)
f.close()
def plotGOESMain(event, start, stop, comp_test='NA'):
'read GOES data from data_dir and plot them. Input: event, interruption start and stop time (e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57)'
#
#--- read radiation zone information
#
radZone = ptrf.readRadZone(event)
#
#--- read GOES data
#
if comp_test == 'test':
file = test_data_dir + event + '_goes.txt'
plot_out = test_goes_dir
else:
file = data_dir + event + '_goes.txt'
plot_out = goes_dir
f = open(file, 'r')
data = [line.strip() for line in f.readlines()]
f.close()
dofy = []
p1 = []
p2 = []
p5 = []
dcnt = 0
for ent in data:
atemp = re.split('\s+|\t+', ent)
btemp = re.split('\.', atemp[0])
if ent and btemp[0].isdigit():
dofy.append(atemp[0])
val1 = float(atemp[1])
if val1 <= 0:
val1 = 1e-5
val2 = float(atemp[2])
if val2 <= 0:
val2 = 1e-5
val3 = float(atemp[3])
if val3 <= 0:
val3 = 1e-5
p1.append(math.log10(val1))
p2.append(math.log10(val2))
p5.append(math.log10(val3))
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- if the interuption go over two years, adjust the ending ydate to that of the previous year
#
if year2 > year1:
chk = 4.0 * int(0.25 * year1)
if chk == year1:
base = 366
else:
base = 365
ydate2 += base
#
#--- plot data
#
if pannelNum == 1:
plotGOES(dofy, p1, p2, p5, ydate1, ydate2, plotStart, plotStop, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes.png'
os.system(cmd)
#
#--- if the interruption period cannot be covered by one plotting panel, create as many panels as we need to cover the period.
#
else:
pstart = plotStart
prange = pannelNum + 1
for i in range(1, prange):
pend = pstart + 5
if i == 1:
plotGOES(dofy, p1, p2, p5, ydate1, 'NA', pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes.png'
os.system(cmd)
elif i == pannelNum:
plotGOES(dofy, p1, p2, p5, 'NA', ydate2, pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes_pt'+ str(i) + '.png'
os.system(cmd)
else:
plotGOES(dofy, p1, p2, p5, 'NA', 'NA', pstart, pend, radZone)
cmd = 'mv ./out.png ' + plot_out + event + '_goes_pt'+ str(i) + '.png'
os.system(cmd)
pstart = pend
def useDataSeeker(startYear, startYdate, stopYear, stopYdate, msid):
"""
extract data using dataseeker.
input: start, stop (e.g., 2012:03:13:22:41)
msid ---- msid
output: data --- two column data (time and msid data)
"""
#
#--- set dataseeker input file
#
(year1, month1, day1, hours1, minute1, second1, ydate1, dom1, sectime1)\
= tcnv.dateFormatConAll(startYear, startYdate)
(year2, month2, day2, hours2, minute2, second2, ydate2, dom2, sectime2)\
= tcnv.dateFormatConAll(stopYear, stopYdate)
#
#--- check a dummy 'test' file exists. it also needs param directory
#
if not os.path.isfile('test'):
fo = open('./test', 'w')
fo.close()
try:
clean_dir('param')
except:
cmd = 'mkdir ./param 2> /dev/null'
os.system(cmd)
mcf.rm_file('./temp_out.fits')
#
#--- name must starts with "_"
#
mc = re.search('deahk', msid.lower())
mc2 = re.search('oobthr', msid.lower())
#
#--- deahk cases
#
if mc is not None:
atemp = re.split('deahk', msid)
val = float(atemp[1])
if val < 17:
name = 'rdb..deahk_temp.' + msid.upper() + '_avg'
else:
name = 'rdb..deahk_elec.' + msid.upper() + '_avg'
#
#--- oobthr cases
#
elif mc2 is not None:
name = 'mtatel..obaheaters_avg._' + msid.lower() + '_avg'
#
#--- special cases (see the list at the top)
#
elif msid.upper() in special_list:
name = msid.upper() + '_AVG'
else:
name = '_' + msid.lower() + '_avg'
#
#--- create dataseeker command
#
cmd1 = '/usr/bin/env PERL5LIB="" '
cmd2 = ' source /home/mta/bin/reset_param; '
cmd2 = ' '
cmd2 = cmd2 + ' /home/ascds/DS.release/bin/dataseeker.pl '
cmd2 = cmd2 + ' infile=test outfile=temp_out.fits '
cmd2 = cmd2 + ' search_crit="columns=' + name
cmd2 = cmd2 + ' timestart=' + str(sectime1)
cmd2 = cmd2 + ' timestop=' + str(sectime2)
cmd2 = cmd2 + ' " loginFile='+ house_keeping + 'loginfile '
cmd = cmd1 + cmd2
bash(cmd, env=ascdsenv)
cmd = 'rm /data/mta/dataseek* 2>/dev/null'
os.system(cmd)
[cols, tbdata] = read_fits_file('./temp_out.fits')
mcf.rm_file('./temp_out.fits')
time = list(tbdata.field('time'));
vals = list(tbdata.field(cols[1]));
data = [time, vals]
return data
def printEachHtml(event, start, stop, gap, stopType):
"""
create indivisual event html page.
input: event --- event name
start --- start time
stop --- stop time
gap --- aount of interrution in sec
stopType --- auto/manual
example: 20031202 2003:12:02:17:31 2003:12:04:14:27 139.8 auto
output: html pages
"""
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1, sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2, sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart,\
plotStart, plotYearStop, plotStop, pannelNum) = itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- check whether we need multiple pannels
#
pannelNum = int((plotStop - plotStart) / 5)
#
#--- choose a template
#
atemp = re.split(':', start)
year = int(atemp[0])
if year < 2011:
file = house_keeping + 'sub_html_template'
elif year < 2014:
file = house_keeping + 'sub_html_template_2011'
else:
file = house_keeping + 'sub_html_template_2014'
#
#--- read the template and start substituting
#
data = open(file).read()
data = re.sub('#header_title#', event, data)
data = re.sub('#main_title#', event, data)
data = re.sub('#sci_run_stop#', start, data)
data = re.sub('#sci_run_start#', stop, data)
data = re.sub('#interruption#', gap, data)
data = re.sub('#trigger#', stopType, data)
noteN = event + '.txt'
data = re.sub('#note_name#', noteN, data)
#
#--- ACA (NOAA) radiation data
#
aceData = event + '_dat.txt'
data = re.sub('#ace_data#', aceData, data)
file = stat_dir + event + '_ace_stat'
try:
stat = open(file).read()
data = re.sub('#ace_table#', stat, data)
line = event + '.png"'
for i in range(2, pannelNum+1):
padd = ' alt="main plot" style="width:100%">\n<br />\n<img src = "../Main_plot/'
padd = padd + event + '_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#ace_plot#', line , data)
except:
pass
#
#---EPHIN data
#
ephData = event + '_eph.txt'
data = re.sub('#eph_data#', ephData, data)
file = stat_dir + event + '_ephin_stat'
try:
stat = open(file).read()
data = re.sub('#eph_table#', stat, data)
line = event + '_eph.png"'
for i in range(2, pannelNum+1):
padd = ' alt="eph plot" style="width:100%">\n<br />\n<img src = "../Ephin_plot/'
padd = padd + event + '_eph_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#eph_plot#', line , data)
except:
pass
#
#---GOES data
#
goesData = event + '_goes.txt'
data = re.sub('#goes_data#', goesData, data)
file = stat_dir + event + '_goes_stat'
try:
stat = open(file).read()
data = re.sub('#goes_table#', stat, data)
line = event + '_goes.png"'
for i in range(2, pannelNum+1):
padd = ' alt="goes plot" style="width:100%"> \n<br />\n<img src = "../GOES_plot/'
padd = padd + event + '_goes_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#goes_plot#', line , data)
if year1 >= 2011:
data = re.sub('GOES-11', 'GOES-15', data)
except:
pass
#
#---XMM data
#
xmmData = event + '_xmm.txt'
data = re.sub('#xmm_data#', xmmData, data)
file = stat_dir + event + '_xmm_stat'
try:
stat = open(file).read()
data = re.sub('#xmm_table#', stat, data)
line = event + '_xmm.png"'
for i in range(2, pannelNum+1):
padd = ' alt="xmm plot" style="width:100%"> \n<br />\n<img src = "../XMM_plot/'
padd = padd + event + '_xmm_pt' + str(i) + '.png '
line = line + padd
data = re.sub('#xmm_plot#', line , data)
except:
pass
#
#--- ACIS
#
fin = open('./acis_list', 'r')
alist = [line.strip() for line in fin.readlines()]
fin.close()
for ent in alist:
mc = re.search(event, ent)
if mc is not None:
atemp = re.split('::', ent)
blist = re.split(':', atemp[1])
break
k = 0
aline = ''
for ent in blist:
aline = aline + "<img src='http://acis.mit.edu/asc/txgif/gifs/"
aline = aline + ent + ".gif' style='width:45%; padding-bottom:30px;'>\n"
k += 1
if k % 2 == 0:
aline = aline + '<br />\n'
data = re.sub('#acis_plot#', aline, data)
#
#--- print the page
#
file = web_dir + 'Html_dir/' + event + '.html'
f = open(file, 'w')
f.write(data)
f.close()
def printEachHtml(event, start, stop, gap, stopType):
'create indivisual event html page. input event name, interruption start/stop time, gap, and type (auto/manual): example: 20031202 2003:12:02:17:31 2003:12:04:14:27 139.8 auto'
#
#--- modify date formats
#
begin = start + ':00'
(year1, month1, date1, hours1, minutes1, seconds1, ydate1, dom1,
sectime1) = tcnv.dateFormatConAll(begin)
end = stop + ':00'
(year2, month2, date2, hours2, minutes2, seconds2, ydate2, dom2,
sectime2) = tcnv.dateFormatConAll(end)
#
#--- find plotting range
#
(pYearStart, periodStart, pYearStop, periodStop, plotYearStart, plotStart, plotYearStop, plotStop, pannelNum) \
= itrf.findCollectingPeriod(year1, ydate1, year2, ydate2)
#
#--- check whether we need multiple pannels
#
pannelNum = int((plotStop - plotStart) / 5)
#
#--- choose a template
#
atemp = re.split(':', start)
year = int(atemp[0])
if year < 2011:
file = house_keeping + 'sub_html_template'
elif year < 2014:
file = house_keeping + 'sub_html_template_2011'
else:
file = house_keeping + 'sub_html_template_2014'
#
#--- read the template and start substituting
#
data = open(file).read()
data = re.sub('#header_title#', event, data)
data = re.sub('#main_title#', event, data)
data = re.sub('#sci_run_stop#', start, data)
data = re.sub('#sci_run_start#', stop, data)
data = re.sub('#interruption#', gap, data)
data = re.sub('#trigger#', stopType, data)
noteN = event + '.txt'
data = re.sub('#note_name#', noteN, data)
#
#--- ACA (NOAA) radiation data
#
aceData = event + '_dat.txt'
data = re.sub('#ace_data#', aceData, data)
file = stat_dir + event + '_ace_stat'
stat = open(file).read()
data = re.sub('#ace_table#', stat, data)
line = event + '.png"'
for i in range(2, pannelNum + 1):
padd = ' alt="main plot" style="width:100%">\n<br />\n<img src = "../Main_plot/' + event + '_pt' + str(
i) + '.png '
line = line + padd
data = re.sub('#ace_plot#', line, data)
#
#---EPHIN data
#
ephData = event + '_eph.txt'
data = re.sub('#eph_data#', ephData, data)
file = stat_dir + event + '_ephin_stat'
stat = open(file).read()
data = re.sub('#eph_table#', stat, data)
line = event + '_eph.png"'
for i in range(2, pannelNum + 1):
padd = ' alt="eph plot" style="width:100%">\n<br />\n<img src = "../Ephin_plot/' + event + '_eph_pt' + str(
i) + '.png '
line = line + padd
data = re.sub('#eph_plot#', line, data)
#
#---GOES data
#
goesData = event + '_goes.txt'
data = re.sub('#goes_data#', goesData, data)
file = stat_dir + event + '_goes_stat'
stat = open(file).read()
data = re.sub('#goes_table#', stat, data)
line = event + '_goes.png"'
for i in range(2, pannelNum + 1):
padd = ' alt="goes plot" style="width:100%"> \n<br />\n<img src = "../GOES_plot/' + event + '_goes_pt' + str(
i) + '.png '
line = line + padd
data = re.sub('#goes_plot#', line, data)
if year1 >= 2011:
data = re.sub('GOES-11', 'GOES-15', data)
file = web_dir + 'Html_dir/' + event + '.html'
f = open(file, 'w')
f.write(data)
f.close()
