def update_main_html():

    [tyear, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")
    
    today = str(mon) + '/' + str(day) + '/ ' +str(tyear)

    syear = tyear
    smon  = mon -1
    if smon < 1:
        smon  = 12
        syear = tyear -1

    lyear = str(syear)

    lmon  = str(smon)
    if smon < 10:
        lmon = '0' + lmon

    line = lmon + '_' + lyear

    file = hosue_keeping + 'template'
    data = open(file, 'r').read()
    data = data.replace('#LATEST#', line)
    data = data.replace('#DATE#',   today)

    file = web_dir + 'exposure.html'
    fo   = open(file, 'w')
    fo.write(data)
    fo.close()
Example #2
0
def plot_steps(time, set1, set2):

    """
    plot tsc and fa movement
    input:  time    --- a list of time in fractional year
            set1    --- a list of tsc value
            set2    --- a list of fa value
    output: monthly_sim.png
    """

#
#--- set plottting range
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
    xmin  = 2000
    xmax  = year + 1
    if mon > 6:
        xmax += 1

    ymin  = 0
    ymax  = max(set1)
    ymax1 = int(1.1 * ymax) + 10

    ymax  = max(set2)
    ymax2 = int(1.1 * ymax) + 10
#
#-- set a few parameters
#
    fsize  = 9
    lsize  = 0
    color  = 'red'
    marker = 'o'
    msize  = 3
    plt.close("all")
    mpl.rcParams['font.size'] = fsize
    props = font_manager.FontProperties(size=fsize)
#
#--- TSC plot
#
    a1 = plt.subplot(121)
    plot_sub(a1, time, set1, xmin, xmax, ymin, ymax1, color, lsize, marker, msize, tline='TSC')

    a1.set_xlabel('Time (year)', size=fsize)
    a1.set_ylabel('TSC Cummulative Moter Dist (x10^4 mm)', size=fsize)
#
#-- FA plot
#
    a2 = plt.subplot(122)
    plot_sub(a2, time, set2, xmin, xmax, ymin, ymax2, color, lsize, marker, msize, tline='FA')

    a2.set_xlabel('Time (year)', size=fsize)
    a2.set_ylabel('FA Cummulative Moter Dist (mm)', size=fsize)
#
#--- save the plot
#
    fig = matplotlib.pyplot.gcf()
    fig.set_size_inches(10.0, 5.0)

    outname = 'monthly_sim.png'
    plt.savefig(outname, format='png', dpi=100)
def print_month_list():

    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    line  = 'Sep99 41 71\n'
    begin = 71
    for iyear in range(1999, year+1):

        if tcnv.isLeapYear(iyear) == 1:
            month_list = month_list2
        else:
            month_list = month_list1

        lyear = str(iyear)
        syear = lyear[2] + lyear[3]
        for month in range(1, 13):
            if iyear == 1999 and  month < 10:
                continue
            if iyear == year and month > mon:
                break

            lmon = tcnv.changeMonthFormat(month)
            time = lmon + syear

            daynum = month_list[month-1]
            end    = daynum + begin
            
            line   = line + time + ' ' + str(begin) + ' ' + str(end) + '\n'
            begin  = end

    fo = open('/data/mta_www/mta_sim/Scripts/SIM_move/Outputs/months', 'w')
    fo.write(line)
    fo.close()
def find_data_collection_interval():

    tlist = tcnv.currentTime(format='UTC')
    tyear = tlist[0]
    tyday = tlist[7]
    tdom  = tcnv.YdateToDOM(tyear, tyday)

    file  = data_dir + 'Disp_dir/hist_ccd3'
    f     = open(file, 'r')
    data  = [line.strip() for line in f.readlines()]
    f.close()

    chk   = 0
    k     = 1
    while(chk == 0):
        atemp = re.split('<>', data[len(data)-k])
        ldom  = atemp[0]
        if mcf.chkNumeric(ldom) == True:
            ldom = int(ldom)
            chk = 1
            break
        else:
            k += 1

    ldom += 1
    return(ldom, tdom)
Example #5
0
def acis_dose_control(year = 'NA', month = 'NA', comp_test = 'NA'):

    """
    monthly acis dose update control script
    input: optional year and month
    """

    if year == 'NA' or month == 'NA':

        (year, mon, day, hours, min, sec, weekday, yday, dst) = tcnv.currentTime('Local')

        month = mon -1
        if month < 1:
            month = 12
            year -= 1
    else:
        year  = int(year)
        month = int(month)

    syear = str(year)
    smon  = str(month)
    if month < 10:
        smon = '0' + smon

#
#--- extract data
#
    getd.acis_dose_get_data(year, month, year, month)
Example #6
0
def add_date_on_html():

    """
    updating the modified date on three html files
    Input:  None
    Outpu:  three htmla pages updated
    """

    current   = tcnv.currentTime(format='Display')

    if level == 1:
            top_level = '/'
    else:
            top_level = '/Lev2/'
    html_file = 'long_term.html'
    plot_out  =  top_level + 'Plots/Plot_long_term/'
    change_date(current, html_file, plot_out)

    html_file = 'past_one_year.html'
    plot_out  =  top_level + 'Plots/Plot_past_year/'
    change_date(current, html_file, plot_out)

    html_file = 'quarter.html'
    plot_out  =  top_level + 'Plots/Plot_quarter/'
    change_date(current, html_file, plot_out)
def hrc_create_dose_map(year = 'NA', mon = 'NA'):

    'extract data, create a center part map/statistics, driving script. if year and month are given, computer for that year/month. Otherwise one month before the current month'


#
#--- if year and mon are not provided, check a month before the current month
#
    if year == 'NA':
        [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')

        lyear = year
        lmon  = mon -1
        if lmon < 1:
            lmon = 12
            lyear -= 1
    else:
        lyear = year
        lmon  = mon
#
#--- ectract data
#
#    getdata.hrc_dose_get_data(lyear, lmon, lyear, lmon)
#
#--- compute stat
#
#    hstat.hrc_dose_extract_stat_data_month(lyear, lmon, cum_dir, mon_dir, data_out)
#
#--- plot data
#
    eplot.hrc_dose_plot_exposure_stat(data_out, plot_dir)
def set_min_max(xdata, ydata, xtime = 0, ybot = -999):

    """
    set plotting range
    Input:  xdata   ---- xdata
            ydata   ---- ydata
            xtime   ---- if it is >0, it set the plotting range from 1999 to the current in year
            ybot    ---- if it is == 0, the ymin will be 0, if the ymin computed is smaller than 0
    Output: [xmin, xmax, ymin, ymax]
    """

    xmin  = min(xdata)
    xmax  = max(xdata)
    xdiff = xmax - xmin
    xmin -= 0.1 * xdiff
    xmax += 0.2 * xdiff

    if xtime > 0:
        xmin  = 1999
        tlist = tcnv.currentTime()
        xmax  = tlist[0] + 1

    ymin  = min(ydata)
    ymax  = max(ydata)
    ydiff = ymax - ymin
    ymin -= 0.1 * ydiff
    ymax += 0.2 * ydiff

    if ybot == 0:
        if ymin < 0:
            ymin = 0

    return [xmin, xmax, ymin, ymax]
Example #9
0
def plot_cum_grating(time, h_in, l_in):
    """
    plot cummulative count rates of hetig and letig insertion. 
    input:  time    --- fractional year
            h_in    --- hetig insertion cummulative count rate (month step)
            l_in    --- letig insertion cummulative count rate 
    output: monthly_grat.prn
    """
#
#--- set x axis plotting range
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
    xmin  = 2000
    xmax  = year + 1
    if mon > 6:
        xmax += 1
#
#--- set y axis plotting range
#
    ymin = 0.0
    ymax = max(h_in)
    ymax2 = max(l_in)
    if ymax2 > ymax:
        ymax = ymax2
    ymax = int(1.1 * ymax) + 10
#
#--- set a few parameters
#
    fsize  = 9
    lsize  = 0
    color  = 'red'
    marker = 'o'
    msize  = 3
    plt.close("all")
    mpl.rcParams['font.size'] = fsize
    props = font_manager.FontProperties(size=fsize)
    plt.subplots_adjust(hspace=0.08, wspace=0.10)
#
#--- HETG Cumulative Count Plots
#
    a1 = plt.subplot(121)           #--- two panel plot: left
    plot_sub(a1, time, h_in, xmin, xmax, ymin, ymax, color, lsize, marker,  msize, tline='HETG')

    a1.set_xlabel('Time (year)', size=fsize)
    a1.set_ylabel('Cumulative Insertion Counts', size=fsize)
#
#--- LETG Cumulative Count Plots
#
    a1 = plt.subplot(122)           #--- two panel plot: right
    plot_sub(a1, time, l_in, xmin, xmax, ymin, ymax, color, lsize, marker,  msize, tline='LETG')

    a1.set_xlabel('Time (year)', size=fsize)
#
#--- save the plot
#
    fig = matplotlib.pyplot.gcf()
    fig.set_size_inches(10.0, 5.0)
    outname = 'monthly_grat.png'
    plt.savefig(outname, format='png', dpi=100)
def update_html():

    """
    create/update error html page for a gvien machine and a user
    input: error_list_<cpu>_<user>, wheren cpu and user are found from the machine running this and account using.
    output: cron_error_<cpu>_<usr>.html in html_dir.
    """

#
#--- find the current time
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')

    syear = str(year)
    smon  = str(mon)
    lmon = tcnv.changeMonthFormat(mon)

    if mon < 10:
        smon = '0' + smon
#
#--- set the file name and a hmtl page name
#
    file = error_log_dir + machine + '_' + user + '_' + smon + '_' + syear
    html = html_dir + 'cron_error_' + machine + '_' + user + '_' + smon + '_' + syear + '.html'
#
#--- start writing the html page
#
    out  = open(html, 'w')
    line = '<!DOCTYPE html>\n'
    line = line + '<html>\n'
    line = line + '<head>\n'
    line = line + '<title>Cron Error Log for ' + user.upper() + ' on ' +  machine.upper() + ': ' + lmon + ' ' + syear + '</title>\n'
    line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
    line = line + '</head>\n'
    line = line + '<body>\n'
    line = line + '<h3 style="padding-bottom: 10px"> Cron Error Log for ' + user.upper() + ' on ' +  machine.upper() + ': ' + lmon + ' ' + syear + '</h3>\n\n'
    line = line + '<hr />\n'
    line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n' 

    out.write(line)
#
#--- write the content of error_list
#
    f    = open(file, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    for ent in data:
        out.write(ent)
        out.write('\n')

    line = '</pre>\n'
    line = line + '<br /><hr /><br />\n'
    line = line + 'Back to <a href="https://cxc.cfa.harvard.edu/mta_days/mta_cron/cron_error_main.html">Top Page</a>\n'
    line = line + '\n</body>\n'
    line = line + '</html>\n'
    out.write(line)

    out.close()
Example #11
0
def create_this_montcreate_this_month():
    """
    create ACA monthly script for the month
    input:  none, but read from ./Template
    outpu:  a file named .update_this_month
    """


    tlist = tcnv.currentTime()

    year  = str(tlist[0])
    syr   = year[2] + year[3]

    mon   = int(float(tlist[1]))
    smon  = str(tlist[1])
    if mon < 10:
        smon = '0' + smon
    lmon  = tcnv.changeMonthFormat(mon)
#
#--- format of: 2015_05
#
    yearmon = year + '_' + smon

#--- format of: MAY15
#
    umonyr  = lmon + syr
    umonyr  = umonyr.upper()
#
#--- format of: may15
#
    lmonyr  = umonyr.lower()


    file = dir_path + 'Template/update_temp'
    f    = open(file, 'r')
    data = f.read()
    f.close()

    data = data.replace('#YEAR_MON#', yearmon)
    data = data.replace('#UMONYR#',   umonyr)
    data = data.replace('#LMONYR#',   lmonyr)


    ofile = out_path + 'update_this_month'
    fo    = open(ofile, 'w')
    fo.write(data)
    fo.close()

    cmd = 'chmod 755 ' + ofile
    os.system(cmd)
#
#--- make output directory
#
    cmd = 'mkdir ' + out_path + '/' + umonyr
    os.system(cmd)
    cmd = 'mkdir ' + out_path + '/' + umonyr + '/Report'
    os.system(cmd)
    cmd = 'mkdir ' + out_path + '/' + umonyr + '/Data'
    os.system(cmd)
Example #12
0
def move_tl_files():
    """
    manage # of trace files in the directories
        the file is kept in /data/mta/Script/Dumps/ for 3 days
        and also gizpped files are save in two different directory
        after 3 days, the files are removed from the main directory, but
        other copies are kept another 6 days.

    input:  none but read from directories
    output: gzipped files in TLfiles/Dumps_mon/IN directories

    """
#
#--- set directory paths
#
    mdir = '/data/mta/Script/Dumps/'
    tdir = '/data/mta/Script/Dumps/TLfiles/'
    ddir = '/data/mta/Script/Dumps/Dumps_mon/IN/'
    sdir = ddir + '/Done/'
#
#--- find today's date in seconds from 1998.1.1
#
    today = int(tcnv.currentTime(format='SEC1998'))
#
#--- set boundaries at 3 days ago and 6 days ago
#
    day3ago = today - 1 * 86400
    day6ago = today - 3 * 86400
#
#--- find trace log file older than 3 days in the main direcotry, remove them
#
    flist = get_file_list(mdir)

    remove_older_files(flist, day3ago)
#
#--- remove trace logs older than 6 days ago from TLfiles directory
#
    flist = get_file_list(tdir)

    remove_older_files(flist, day6ago)
#
#--- remove trace logs older than 6 days ago from Dumps_mon/IN/Done directory
#
    flist = get_file_list(sdir)

    remove_older_files(flist, day6ago)
#
#--- now copy new files to appropriate directory
#
    find_new_files(mdir, ddir, '*CCDM*')
    find_new_files(mdir, ddir, '*PCAD*')
    find_new_files(mdir, ddir, '*ACIS*')
    find_new_files(mdir, ddir, '*IRU*')
    find_new_files(mdir, ddir, '*MUPS2*')

    find_new_files(mdir, tdir, '*EPHIN*',   fzip=1)
    find_new_files(mdir, tdir, '*ELBILOW*', fzip=1)
    find_new_files(mdir, tdir, '*MUPS*',    fzip=1)
Example #13
0
def plot_dusk_result(diskName, duskName, nameList, pastData):

#
#--- find the disk capacity of the given disk
#
    disk_capacity = diskCapacity(diskName)

#
#--- read the output from dusk
#
    line = run_dir + duskName
    f    = open(line, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    capacity = {}               #---- make a dictionary
    for ent in data:
        atemp = re.split('\s+|\t+', ent)
        try:
            val = 100.0 * float(atemp[0]) /disk_capacity
            val =  round(val, 2)
            capacity[atemp[1]] = val 
        except:
            pass
#
#--- today's date
#
    today   = tcnv.currentTime('local')
    year    = today[0]
    month   = today[1]
    day     = today[2]
    hours   = today[3]
    minutes = today[4]
    seconds = today[5]
#
#--- convert to dom
#
    dom   = tcnv.findDOM(year, month, day, hours, minutes,seconds)
    dom   = round(dom, 2)
#
#--- append the new data to the data table
#
    f   = open(pastData, 'a')
    f.write(str(dom))
    for dName in nameList:
        f.write(':')
        string = str(capacity[dName])
        f.write(string)
    f.write('\n')
    f.close()

#
#---- start plotting history data
#
    try:
        plot_history_trend(diskName, duskName, nameList, pastData)
    except:
        pass
def run_all_sim_script():

    grat.plot_grat_movement()
    sim.plot_sim_movement()

    [year, month, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
    month -= 1
    if month < 1:
        month = 12
        year -= 1

    aor.plot_aorwspd(year, month)
def find_time_interval(test='NO', month=4, year=2015):
    """
    create start and stop time and output directory name
    input:  test    --- if it is test, yes. default: 'NO'
            month   --- test month input. default: 4. it is ignored if not test.
            year    --- test year input.  default: 2015. it is ignored if not test.
    output: start   --- start time in the format of mm/01/yy (e.g. 03/01/15)
            stop    --- stop time
            dir     --- output directory name in the format of <Mon><yy> (e.g. Jan15)
    """

#
#--- for the case if you want to run the test, there is option...
#
    if test == 'NO':
#
#--- this function gives you the current time. year in the format of 2015, and month in 4
#
        [year, month, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
#
#--- we need start and stop times in the formats of 03/01/15, 04/01/15
#
    year_start = year
    year_end   = year

    this_month = str(month)
    if month < 10:
        this_month = '0' + this_month

    temp_month = month -1
    if temp_month < 1:
        temp_month = 12
        year_start -= 1
    last_month = str(temp_month)
    if temp_month < 10:
        last_month = '0' + last_month

    temp       = str(year_start)
    year_start = temp[2] + temp[3]

    temp       = str(year_end)
    year_end   = temp[2] + temp[3]

    start = last_month + '/01/' + year_start
    stop  = this_month + '/01/' + year_end
#
#--- create the output directory name (format example: Apr15)
#
    cmonth = tcnv.changeMonthFormat(temp_month)
    dir    = cmonth + year_start

    return (start, stop, dir)
Example #16
0
def find_available_deph_data():
    """
    create a list of potential new data file name
    input:  none, but read from /dsops/GOT/aux/DEPH.dir/
    output: cdata   --- a list of the data file names
    """
#
#--- find current time
#
    ttemp = tcnv.currentTime()
    year  = int(ttemp[0])
    ydate = int(ttemp[7])

    syear = str(year)
    tyear = syear[2] + syear[3]
#
#--- first 20 days of the year, we also check the last year input data
#
    if ydate < 20:
        lyear  = year -1
        slyear = str(lyear)
        ltyear = slyear[2] + slyear[3]

        cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + ltyear + '*.EPH >  ' + zspace
        os.system(cmd)
        cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + tyear  + '*.EPH >> ' + zspace
        try:
            os.system(cmd)
        except:
            pass
    else:
        cmd = 'ls /dsops/GOT/aux/DEPH.dir/DE' + tyear  + '*.EPH >  ' + zspace
        try:
            os.system(cmd)
        except:
            pass

    try:
        f     = open(zspace, 'r')
        cdata = [line.strip() for line in f.readlines()]
        f.close()
    except:
        cdata = []

    mcf.rm_file(zspace)

    return cdata
def make_data_list(year, mon, day, tail):
    """
    make a data list from the last entry date to the most current data
    input:  year    --- the year of the last entry
            mon     --- the month of the last entry
            day     --- the day of month of the last entry
            tail    --- the suffix of the data file 
    output  dlist   --- a list of data names
    """
#
#--- convert the date into seconds from 1998.1.1
#
    dst   = tcnv.convertDateToTime2(year, mon, day)
#
#--- find today's time
#
    today = tcnv.currentTime()
    cyear = today[0]
    cmon  = today[1]
    cday  = today[2]
    cdst  = tcnv.convertDateToTime2(cyear, cmon, cday)

    dlist = []
#
#--- check the current date is larger than the date indicated
#--- if so find how many days between and retrieve data for each day
#
    if cdst > dst:
        step = int(( cdst - dst)/86400)
        if step >= 1:
            head  = make_header(year, mon, day)
            name  = head + tail
            dlist.append(name)

        for i in range(2, step):
            sdate = int(cdst - 86400.0 * i)
            out   = tcnv.axTimeMTA(sdate)
            atemp = re.split(':', out)
            year  = int(atemp[0])
            ydate = int(atemp[1])

            [mon, day] = tcnv.changeYdateToMonDate(year, ydate)
            head  = make_header(year, mon, day)
            name  = head + tail
            dlist.append(name)

    return dlist
def update_plt_html_date(comp_test = 'NA'):

    """
    update html pages for plots; just replacing date
    no input, but get the list from plot_dir
    """
    (lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday, dst) = tcnv.currentTime('Local')

    date = 'Last Update: ' + str(lmon) + '/' + str(lday) + '/' + str(lyear)


    if comp_test == 'test':
        cmd  = 'ls ' + test_plot_dir  + '/*html>./ztemp' 
    else:
        cmd  = 'ls ' + plot_dir  + '/*html>./ztemp' 

    os.system(cmd)
    f    = open('./ztemp', 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    os.system('rm ./ztemp')

    for ent in data:
        f    = open(ent, 'r')
        hdat = [line.strip() for line in f.readlines()]
        f.close()
        f    = open('./temp', 'w')

        for oline in hdat:
            m = re.search('Last Update', oline)
            if m is not None:
                f.write(date)
                f.write('\n')

            else:
                f.write(oline)
                f.write('\n')

        f.close() 

        cmd = 'mv ./temp ' + ent
        os.system(cmd)
Example #19
0
def set_min_max(ydata):
    """
    set plotting range
    Input:  ydata   ---- ydata
    Output: [xmin, xmax, ymin, ymax]
    """

    xmin = 1999
    tlist = tcnv.currentTime()
    xmax = tlist[0] + 1

    ymin = min(ydata)
    ymax = max(ydata)
    ydiff = ymax - ymin
    ymin -= 0.1 * ydiff
    ymax += 0.2 * ydiff
    if ymin < 0:
        ymin = 0

    return [xmin, xmax, ymin, ymax]
def ccd_comb_plot(choice):

    """
    a control script to create plots
    Input:  choice      --- if check, you will be asked to provide year and month
                            all oter cases, the script will use the previous month
    Output: png formated plotting files
    """
#
#--- find today's date, and set a few thing needed to set output directory and file name
#
    if choice == 'check':
        year = raw_input("Year: ")
        mon  = raw_input("Month: ")
        year = int(float(year))
        mon  = int(float(mon))
    else:
        [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    syear  = str(year)
    smonth = str(mon)
    if mon < 10:
        smonth = '0'+ smonth

    lyear = year 
    lmon  = mon - 1
    if lmon < 1:
        lmon   = 12
        lyear -= 1

    slyear  = str(lyear)
    slmonth = str(lmon)
    if lmon < 10:
        slmonth = '0' + slmonth
#
#--- monthly plot
#
    dlist    = collect_data_file_names('month')
    plot_out = './'
    header   =  'month_plot_ccd'
    plot_data(dlist, plot_out, header, yr=slyear, mo=slmonth,  psize=2.5, xunit='ydate')
Example #21
0
def update_bias_html():

    """
    pdate bias_home.html page
    Input: None but read from:
            <house_keeping>/bias_home.html
    Output: <web_dir>/bias_home.html
    """
#
#--- find today's date
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    lmon = str(mon)
    if mon < 10:
        lmon = '0' + lmon
    lday = str(day)
    if day < 10:
        lday = '0' + lday
#
#--- line to replace
#
    newdate = "Last Upate: " + lmon + '/' + lday + '/' + str(year)
#
#--- read the template
#
    line = house_keeping + 'bias_home.html'
    data = mcf.readFile(line)
#
#--- print out
#
    outfile = web_dir + 'bias_home.html'
    fo   = open(outfile, 'w')
    for ent in data:
        m = re.search('Last Update', ent)
        if m is not None:
            fo.write(newdate)
        else:
            fo.write(ent)
        fo.write('\n')
    fo.close()
def update_plt_html_date(comp_test='NA'):
    """
    update html pages for plots; just replacing date
    no input, but get the list from plot_dir
    """
    (lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday,
     dst) = tcnv.currentTime('Local')

    date = 'Last Update: ' + str(lmon) + '/' + str(lday) + '/' + str(lyear)

    if comp_test == 'test':
        cmd = 'ls ' + test_plot_dir + '/*html>./ztemp'
    else:
        cmd = 'ls ' + plot_dir + '/*html>./ztemp'

    os.system(cmd)
    f = open('./ztemp', 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    os.system('rm ./ztemp')

    for ent in data:
        f = open(ent, 'r')
        hdat = [line.strip() for line in f.readlines()]
        f.close()
        f = open('./temp', 'w')

        for oline in hdat:
            m = re.search('Last Update', oline)
            if m is not None:
                f.write(date)
                f.write('\n')

            else:
                f.write(oline)
                f.write('\n')

        f.close()

        cmd = 'mv ./temp ' + ent
        os.system(cmd)
Example #23
0
def set_min_max(ydata):

    """
    set plotting range
    Input:  ydata   ---- ydata
    Output: [xmin, xmax, ymin, ymax]
    """

    xmin  = 1999
    tlist = tcnv.currentTime()
    xmax  = tlist[0] + 1

    ymin  = min(ydata)
    ymax  = max(ydata)
    ydiff = ymax - ymin
    ymin -= 0.1 * ydiff
    ymax += 0.2 * ydiff
    if ymin < 0:
        ymin = 0

    return [xmin, xmax, ymin, ymax]
def dose_map_creation(year='', mon=''):
    """
    create acis and hrc dose maps
    input:  year    --- year; if it is "", the year of the last month will be used
            mon     --- month;if it is "", the month value of the last month is used
    output: /data/mta/www/mta_max_exp/Images/*.png,     e.g., ACSI_11_2016_i2.png
    """
#
#--- if year and month are not given, use the last month's month and year value
#
    if year == '':
#
#--- find today's date
#
        ltime = tcnv.currentTime()
        year  = ltime[0]
        lyear = str(year)                           #--- '2016'
#
#--- set the last month's month and year
#
        mon   = ltime[1] -1    

        if mon < 1:
            mon   = 12
            year -= 1

    cmon   = str(mon)
    if mon < 10:
        cmon = '0' + cmon                           #--- e.g. 03 or 11

    lmon   = tcnv.changeMonthFormat(mon)            #--- e.g. Mar or Nov

    lmonyr = lmon.lower() + lyear[2] + lyear[3]     #--- e.g. jan16
    lm_y   = cmon + '_' + lyear                     #--- e.g. 03_2016
#
#--- create clean acis and hrc exposure maps using ds9
#
    run_exposure_maps(lyear, cmon)
Example #25
0
def dose_map_creation(year='', mon=''):
    """
    create acis and hrc dose maps
    input:  year    --- year; if it is "", the year of the last month will be used
            mon     --- month;if it is "", the month value of the last month is used
    output: /data/mta/www/mta_max_exp/Images/*.png,     e.g., ACSI_11_2016_i2.png
    """
    #
    #--- if year and month are not given, use the last month's month and year value
    #
    if year == '':
        #
        #--- find today's date
        #
        ltime = tcnv.currentTime()
        year = ltime[0]
        lyear = str(year)  #--- '2016'
        #
        #--- set the last month's month and year
        #
        mon = ltime[1] - 1

        if mon < 1:
            mon = 12
            year -= 1

    cmon = str(mon)
    if mon < 10:
        cmon = '0' + cmon  #--- e.g. 03 or 11

    lmon = tcnv.changeMonthFormat(mon)  #--- e.g. Mar or Nov

    lmonyr = lmon.lower() + lyear[2] + lyear[3]  #--- e.g. jan16
    lm_y = cmon + '_' + lyear  #--- e.g. 03_2016
    #
    #--- create clean acis and hrc exposure maps using ds9
    #
    run_exposure_maps(lyear, cmon)
Example #26
0
def add_html_page(ptype, plot_out,  yr, mo):

    """
    update/add html page to Plot directory
    Input:  ptype       --- indiecator of which html page to be updated
            plot_out    --- a directory where the html page is updated/created
            yr          --- a year of the file
            mo          --- a month of the file
    Output: either month.html or year.hmtl in an appropriate directory
    """

    current = tcnv.currentTime(format='Display')
    lmon    = ''

    if ptype == 'month':
        ofile  = plot_out + 'month.html'
        lmon   = tcnv.changeMonthFormat(int(mo))
        if level == 2:
            file = house_keeping + 'month2.html'
        else:
            file = house_keeping + 'month.html'

    elif ptype == 'year':
        ofile  = plot_out + 'year.html'
        if level == 2:
            file = house_keeping + 'year2.html'
        else:
            file = house_keeping + 'year.html'

    text = open(file, 'r').read()
    text = text.replace('#YEAR#',  yr)
    text = text.replace('#MONTH#', lmon)
    text = text.replace('#DATE#',  current)
    
    f     = open(ofile, 'w')
    f.write(text)
    f.close()
Example #27
0
def add_date_on_html():
    """
    updating the modified date on three html files
    Input:  None
    Outpu:  three htmla pages updated
    """

    current = tcnv.currentTime(format='Display')

    if level == 1:
        top_level = '/'
    else:
        top_level = '/Lev2/'
    html_file = 'long_term.html'
    plot_out = top_level + 'Plots/Plot_long_term/'
    change_date(current, html_file, plot_out)

    html_file = 'past_one_year.html'
    plot_out = top_level + 'Plots/Plot_past_year/'
    change_date(current, html_file, plot_out)

    html_file = 'quarter.html'
    plot_out = top_level + 'Plots/Plot_quarter/'
    change_date(current, html_file, plot_out)
Example #28
0
def update_datatable(per0, per1, per2, per3, per4, per5, per6, per7, per8):
    """
    this function appends the newest data to the disk space data table
    Input: per0 ... per5: new measures for each disk. currently per3 is empty
    Output: <data_out>/disk_space_data (updated)
    """

    #
    #--- find out today's date in Local time frame
    #
    today = tcnv.currentTime('local')
    year = today[0]
    month = today[1]
    day = today[2]
    hours = today[3]
    minutes = today[4]
    seconds = today[5]
    #
    #--- convert to dom
    #
    dom = tcnv.findDOM(year, month, day, hours, minutes, seconds)
    dom = round(dom, 3)
    #
    #--- today's data
    #
    line = str(dom) + '\t' + str(per0) + '\t' + str(per1) + '\t' + str(
        per2) + '\t' + str(per4) + '\t' + str(per5) + '\t' + str(
            per6) + '\t' + str(per7) + '\t' + str(per8) + '\t'
    #
    #--- append to the data table
    #
    file = data_out + 'disk_space_data'
    f = open(file, 'a')
    f.write(line)
    f.write("\n")
    f.close()
Example #29
0
def check_obs_status(obsid):
    """
    check obsid is eligible to be added to the curren list. 
    input: obsid. if it is either 'scheduled', 'unobserved', or 'observed' status and if it is obsverd
    less than 30 day ago, it is eliible. 
    """
    #
    #--- set limit to the last 30  days
    #
    [year, mon, day, hours, min, sec, weekday, yday,
     dst] = tcnv.currentTime('LOCAL')
    tdom = tcnv.findDOM(year, mon, day, 0, 0, 0)
    dom_limit = tdom - 30

    #
    #--- extract basic information
    #
    monitor = []
    groupid = []
    try:
        (group_id, pre_id, pre_min_lead, pre_max_lead, grating, type, instrument, obs_ao_str, status, \
        seq_nbr, ocat_propid, soe_st_sched_date, lts_lt_plan,targname) = sql.get_target_info(int(obsid), monitor,groupid)

        if soe_st_sched_date is not None:
            date = soe_st_sched_date
            chk = 1
        elif lts_lt_plan is not None:
            date = lts_lt_plan
            chk = 1
        else:
            date = 'NA'
            chk = 0
    except:
        date = 'NA'
        chk = 0
#
#--- check status change
#
    if chk == 1:
        if status == 'scheduled' or status == 'unobserved' or status == 'observed':
            #
            #--- recompute with updated date
            #
            if date == 'NA':
                dom = 'NA'
            else:
                temp = re.split('\s+', str(date))
                omon = tcnv.changeMonthFormat(temp[0])
                oday = int(temp[1])
                oyear = int(temp[2])
                dom = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)

            if dom > dom_limst:
                chk = 1
            else:
                chk = 0

        else:
            chk = 0

    return chk
Example #30
0
def collect_data_file_names(period, syear=2000, smonth=1, eyear=2000, emonth=12):

    """
    for a given period, create a list of directory names
    Input:  period   --- indicator of which peirod, "month", "quarter", "year", "lyear", "full", and "check'"
            if period == 'check', then you need to give a period in year and month
            syear    --- year of the starting date
            smonth   --- month of the starting date
            eyear    --- year of the ending date
            emonth   --- month of the ending date
    Output  data_lst --- a list of the directory names
    """
    
#
#--- find today's date
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    data_list = []

#
#--- find the last month 
#
    if period == 'month':
        mon -= 1
        if mon < 1:
            mon = 12
            year -= 1

        if mon < 10:
            cmon = '0' + str(mon)
        else: 
            cmon = str(mon)

        dfile = data_dir + 'Data_' + str(year) + '_' + cmon
        data_list.append(dfile)
#
#--- find the last three months 
#
    if period == 'quarter':
        for i in range(1, 4):
            lyear = year
            month = mon -i
            if month < 1:
                month = 12 + month
                lyear = year -1

            if month < 10:
                cmon = '0' + str(month)
            else: 
                cmon = str(month)

            dfile = data_dir + 'Data_' + str(lyear) + '_' + cmon
            data_list.append(dfile)
#
#--- find data for the last one year (ending the last month)
#
    elif period == 'year':
        
        cnt = 0
        if mon > 1:
            for i in range(1, mon):
                if i < 10:
                    cmon = '0' + str(i)
                else:
                    cmon = str(i)
                dfile = data_dir + 'Data_' + str(year) + '_' + cmon
                data_list.append(dfile)
                cnt += 1
        if cnt < 11:
            year -= 1
            for i in range(mon, 13):
                if i < 10:
                    cmon = '0' + str(i)
                else:
                    cmon = str(i)
                dfile = data_dir + 'Data_' + str(year) + '_' + cmon
                data_list.append(dfile)
#
#--- fill the list with the past year's data
#
    elif period == 'lyear':
        year -= 1
        for i in range(1, 13):
            if i < 10:
                cmon = '0' + str(i)
            else:
                cmon = str(i)
            dfile = data_dir + 'Data_' + str(year) + '_' + cmon
            data_list.append(dfile)
#
#--- fill the list with the entire data
#
    elif period == 'full':
        for iyear in range(2000, year+1):
            for i in range (1, 13):
                if i < 10:
                    cmon = '0' + str(i)
                else:
                    cmon = str(i)
                dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
                data_list.append(dfile)
#
#--- if the period is given, use them
#
    elif period == 'check':
        syear  = int(syear)
        eyear  = int(eyear)
        smonth = int(smonth)
        emonth = int(emonth)
        if syear == eyear:
            for i in range(smonth, emonth+1):
                if i < 10:
                    cmon = '0' + str(i)
                else:
                    cmon = str(i)
                dfile = data_dir + 'Data_' + str(syear) + '_' + cmon
                data_list.append(dfile)

        elif syear < eyear:
            for iyear in range(syear, eyear+1):
                if iyear == syear:
                    for month in range(smonth, 13):
                        if i < 10:
                            cmon = '0' + str(i)
                        else:
                            cmon = str(i)
                        dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
                        data_list.append(dfile)
                elif iyear == eyear:
                    for month in range(1, emonth+1):
                        if i < 10:
                            cmon = '0' + str(i)
                        else:
                            cmon = str(i)
                        dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
                        data_list.append(dfile)
                else:
                    for month in range(1, 13):
                        if i < 10:
                            cmon = '0' + str(i)
                        else:
                            cmon = str(i)
                        dfile = data_dir + 'Data_' + str(iyear) + '_' + cmon
                        data_list.append(dfile)

    return data_list
def printHtml(indir,outdir,  hrc, date, year,month,mean_acc,std_acc,min_acc,min_apos,max_acc,max_apos, asig1, asig2, asig3,mean_dff,  \
                std_dff,min_dff,min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):

    'create HTML page to display HRC historical data.'


    [tyear, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local")

    smon = str(mon)
    if mon < 10:
        smon = '0' + smon

    sday = str(day)
    if day < 10:
        sday = '0' + sday

    outdir = outdir + '/' + hrc + '.html'

    f = open(outdir, 'w')
#
#--- this is a html 5 document
#
    f.write('<!DOCTYPE html>\n')
    f.write('<html>\n')
    f.write('<head>\n')

    f.write("<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />\n")

    f.write("<style  type='text/css'>\n")
    f.write("table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n")
    f.write("a:link {color:#00CCFF;}\n")
    f.write("a:visited {color:yellow;}\n")
    f.write("td{text-align:center;padding:8px}\n")
    f.write("</style>\n")


    if hrc == 'hrci':
        hname = 'HRC I'
        wname = 'HRCI'
    else:
        hname = 'HRC S'
        wname = 'HRCS'

    line = '<title>' + hname + ' History Data</title>\n'
    f.write(line)
    f.write("</head>\n")

    f.write('<body style="color:white;background-color:black">\n')
    line = '<h2 style="text-align:center">Data: ' + hname + '</h2>\n'
    f.write(line)

    f.write("<div style='padding-bottom:30px'>\n")
    f.write('<table border=1>\n')
    f.write('<tr><th>&#160;</th><th>&#160;</th><th colspan=11>Monlthy</th><th colspan=11>Cumulative</th></tr>\n')
    f.write('<tr style="color:yellow"><th>Year</th><th>Month</th>\n')
    f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n')
    f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n')

    for i in range(0, len(date)):

        smonth = str(month[i])
        if month[i] < 10:
            smonth = '0' + smonth

        cmonth = tcnv.changeMonthFormat(month[i])        #---- converting digit to letters, i.e. 1 to Jan

#
#--- monthly HRC dose data
#

        if mean_dff[i] == 0 and std_dff[i] == 0:

            line = '<tr><td>%d</td><td>%d</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n' % (year[i], month[i])
            f.write(line)
            f.write('<td>No Data</td><td>No Image</td>\n')
        else:
            line = '<tr><td>%d</td><td>%d</td><td>%4.4f</td><td>%4.4f</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td>\n' \
                    % (year[i], month[i], mean_dff[i], std_dff[i], min_dff[i],min_dpos[i], max_dff[i], max_dpos[i],dsig1[i], dsig2[i], dsig3[i])
            f.write(line)

            fname = wname + '_' + smonth + '_' + str(year[i]) + '.fits.gz'
            line  = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/' + fname +'">fits</a></td>\n'
            f.write(line)
            fname = wname + '_' + smonth + '_' + str(year[i]) + '.png'
            line  = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + fname + '">map</a></td>\n'
            f.write(line)

#
#---- cumulative HRC dose data
#
        line = '<td>%4.4f</td><td>%4.4f</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td><td>%s</td><td>%4.1f</td>\n' \
                    % (mean_acc[i], std_acc[i], min_acc[i], min_apos[i], max_acc[i], max_apos[i], asig1[i], asig2[i], asig3[i])

        f.write(line)
        fname = wname + '_08_1999_' + smonth + '_' + str(year[i]) + '.fits.gz'
        line  = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/' + fname +'">fits</a></td>\n'
        f.write(line)
        fname = wname + '_08_1999_' + smonth + '_' + str(year[i]) + '.png'
        line  = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + fname + '">map</a></td>\n\n'
        f.write(line)

#
#--- put header every new year so that we can read data easier
#
        if month[i] % 12 == 0 and i != (len(date)-1):
            f.write('\n<tr style="color:yellow"><th>Year</th><th>Month</th><th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n')
            f.write('<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th><th>Max</th><th>Max Position</th><th>68% Level</th><th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n')

    f.write('</table>\n\n')
    f.write("</div>\n")
    f.write('<hr />\n')

    line = '<p style="padding-top:10px;padding-bottom:10px"><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + sday + '/' + str(tyear) + '</strong></p>\n'
    f.write(line)

    line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
    f.write(line)
    f.write('</body>\n')
    f.write('</html>\n')

    f.close()    
def check_cron():

    """
    find new error messages from cron log files for a given machine and a given user.
    this script send out email if it finds new error message. The log files are clean up 1 of every month
    and the same message could be send out if there are still the same error messages occur.
    """
#
#--- error_log name
#
    error_logs = error_log_dir + 'error_list_' + machine + '_' + user

#
#--- find cron file names for this machine for this user
#
    cron_file_name = extract_cron_file_name()

#
#--- check today's date. if it is 1st of the month, move the old error_list to archive form.
#

    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')

    if day == 1:
        lyear = year
        lmon  = mon - 1
        if lmon < 1:
            lmon = 12
            lyear -= 1

        error_logs_old = error_logs + '_' + str(lmon) + '_' + str(lyear)
        cmd = 'mv ' + error_logs + ' ' + error_logs_old
        os.system(cmd)
        cmd = 'mv '+ error_logs_old + ' ' + error_log_dir + 'Past_logs/.'
        os.system(cmd)

        error_dict = {}
    else:
#
#--- read existing error list
#
        error_dict = {}
        try:
            f    = open(error_logs, 'r')
            data = [line.strip() for line in f.readlines()]
            f.close()
            for ent in data:
                atemp = re.split('\<:\>', ent)
                content = []
                for i in range(1, len(atemp)):
                    content.append(atemp[i])
    
                error_dict[atemp[0]] = content
        except:
            pass

#
#--- set which Log location to check (depends on user.)
#
    dir_loc = '/home/' + user + '/Logs/'

#
#--- find names of files and directories in the Logs directory
#
    cmd = 'ls -lrtd ' + dir_loc + '/* >' + tempout
    os.system(cmd)
    f    = open(tempout, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    cmd = 'rm ' + tempout
    os.system(cmd)

    cron_list =[]
    for ent in data:
        atemp = re.split('\s+|\t+', ent)
        m1    = re.search('d',   atemp[0])
        m2    = re.search('Past_logs', ent)
#
#--- if it is a directory other than Past_logs, find file names in that directory
#
        if (m1 is not None) and (m2 is None):
            cmd = 'ls ' + atemp[8] + '/* > ' + tempout
            os.system(cmd)
            f    = open(tempout, 'r')
            data2 = [line.strip() for line in f.readlines()]
            f.close()
            cmd = 'rm ' + tempout
            os.system(cmd)

            for ent2 in data2:
                cron_list.append(ent2)
#
#--- files in Logs directory level
#
        elif m2 is None:
            cron_list.append(atemp[8])

    new_error_dict = {}
    for file in cron_list:
#
#--- check whether this error message belongs to this machine (and the user)
#
        mchk = 0
        for comp in cron_file_name:
            m = re.search(comp, file)

            if m is not None:
                mchk = 1

        if mchk > 0:
#
#--- check whether the file has any error messages 
#
            error_list = find_error(file)
            if len(error_list) > 0:
#
#--- if there are error messages, compare them to the previous record, and if it is new append to the record.
#
                try:
                    prev_list = error_dict[file]
                    new_error = []
                    for ent in error_list:
                        sent = "".join(ent.split())             #---- removing all white spaces
                        chk = 0
                        for comp in prev_list:
                            scomp = "".join(comp.split())
                            if sent == scomp:
                                chk = 1
    
                        if chk ==  0:
                            prev_list.append(ent)
                            new_error.append(ent)
    
                    if len(new_error) > 0:
                        error_dict[file]     = prev_list
                        new_error_dict[file] = new_error 
                except:
#
#--- there is no previous error_list entry: so all error messages are new and log them
#
                    error_dict[file]     = error_list
                    new_error_dict[file] = error_list


#
#--- update error logs
#
    old_log = error_logs + '~'
    cmd     = 'mv ' + error_logs + ' ' + old_log
    os.system(cmd)

    f = open(error_logs, 'w')
    for key in error_dict:
        line = key
        for e_ent in error_dict[key]:
            line = line + '<:>' + e_ent
        line  = line + '\n'
        f.write(line)

    f.close()

#
#---if new error messages are found; notify to a list of users
#
    chk = 0
    f = open(tempout, 'w')
    for key in new_error_dict:
        chk += 1
        line = key + '\n'
        f.write(line)
        for ent in new_error_dict[key]:
            line = '\t' + ent + '\n'
            f.write(line)

        f.write('\n')

    f.close()

    if chk > 0:
        cmd = 'cat ' + tempout + ' | mailx -s "Subject: Cron Error : ' + user + ' on ' + machine + '"  ' + email_list
        os.system(cmd)
#
#--- add the error message to a recored 
#
        add_to_log()
#
#--- update html page
#
        update_html()
        update_main_html()


    else:
#
#--- if there is no error, notify that fact to admin
#
        f = open(tempout, 'w')
        line = '\nNo error is found today on ' + machine + ' by a user ' + user + '.\n'
        f.write(line)
        f.close()
        cmd = 'cat ' + tempout + ' | mailx -s "Subject: No Cron Error : ' + user + ' on ' + machine + '" ' + admin
        os.system(cmd)


    cmd = 'rm ' + tempout
    os.system(cmd)
Example #33
0
def acis_dose_control(year = 'NA', month = 'NA', comp_test = 'NA'):

    """
    monthly acis dose update control script
    input: optional year and month
    """

    if year == 'NA' or month == 'NA':

        (year, mon, day, hours, min, sec, weekday, yday, dst) = tcnv.currentTime('Local')

        month = mon -1
        if month < 1:
            month = 12
            year -= 1
    else:
        year  = int(year)
        month = int(month)

    syear = str(year)
    smon  = str(month)
    if month < 10:
        smon = '0' + smon

#
#--- extract data
#
    getd.acis_dose_get_data(year, month, year, month)
#
#--- create cumulative data and sectioned data for bot month and cumulative data
#
    file = 'ACIS_' + smon + '_' + syear + '.fits'
    file2= 'ACIS_' + smon + '_' + syear + '.fits.gz'

    chk = 0
    for test in os.listdir('./'):
        if fnmatch.fnmatch(test, file2):
            chk = 1
            break

    if chk == 0:
       cuml.acis_create_cumulative(file, comp_test)
       pass
    else:
       pass
       cuml.acis_create_cumulative(file2, comp_test)
#
#--- compute statistics
#
    astat.acis_dose_extract_stat_data_month(year, month, comp_test)
#
#--- plot data
#
    aplot.acis_dose_plot_exposure_stat(clean='Yes', comp_test=comp_test)
#
#--- create images (you need to use ds9 to create a better image)
#
    aimg.create_acis_maps(year, month, comp_test)
#
#--- update html pages
#
    ahtml.acis_dose_make_data_html(comp_test = comp_test)

#
#--- print monthly output
#
    arport.acis_dose_monthly_report()
Example #34
0
def get_data(ccd_list, out):

    """
    read out data from the full cti data table and creates monthly report data table
    Input: ccd_list     --- a list of ccds which you want to read the data
           out          --- a type of the data, "image", "spec", or "back"
           data are read from "/data/mta/Script/ACIS/CTI/DATA/..."
    Output: xSets       --- a list of lists of x values of each ccd
            ySets       --- a list of lists of y values of each ccd
            eSets       --- a list of lists of y error of each ccd
            ./Data/ccd<ccd>_data: monthly averaged cti for monthly report
    """
#
#--- read intercept adjusting table
#
    al_factors = read_correction_factor('al')
    mn_factors = read_correction_factor('mn')
    ti_factors = read_correction_factor('ti')
#
#--- find today's date; ctime[0] is year and ctime[1] is month
#
    ctime = tcnv.currentTime()
    lyear = int(ctime[0])
    lmonth= int(ctime[1])
#
#--- set dimension of the array
#
    c_cnt = 12 * (int(ctime[0]) - 2000) + int(ctime[1])
    d_cnt = len(ccd_list)
#
#--- for none backside CCDs, we use detrended data sets
#
    if out == 'back':
        dir = '/data/mta/Script/ACIS/CTI/Data/Data_adjust/'
#
#--- vadd to adjust the mean position of CTI
#
    else:
        dir = '/data/mta/Script/ACIS/CTI/Data/Det_Data_adjust/'

    xSets = []
    ySets = []
    eSets = []
#
#-- go around each ccds
#
    for i in range(0, d_cnt):
        ccd = ccd_list[i]
#
#--- set cti data array and error array
#
        avals = [0 for x in range(0, c_cnt)]
        sum   = [0 for x in range(0, c_cnt)]
        sum2  = [0 for x in range(0, c_cnt)]
#
#--- go around all lines
#
        for elm in ('mn'):
            vadd = 0
            if elm == 'al':
                corrections = al_factors
                vadd = 0
                for k in range(0, 4):
                    vadd += corrections[k][ccd]
                vadd /= 4.0
            elif elm == 'ti':
                correcitons = ti_factors
            else:
                corrections = mn_factors
                for k in range(0, 4):
                    vadd += corrections[k][ccd]
                vadd /= 4.0

            file = dir + 'mn_ccd' + str(ccd)
            f    = open(file, 'r')
            data = [line.strip() for line in f.readlines()]
            f.close()

            for ent in data:
                atemp = re.split('\t+|\s+', ent)
                btemp = re.split('-', atemp[0])
#
#--- find the row that you want to add this data 
#
                pos   = 12 * (int(btemp[0]) - 2000) + int(btemp[1]) - 1

                for k in range(1, 5):
                    ctemp = re.split('\+\-', atemp[k])
                    val  = float(ctemp[0])
                    if val > 0 and val < yupper:
#
#--- correct the value so that all data points have about the same base line
#
                        val -= (corrections[k-1][ccd])
                        val += vadd
                        err  = float(ctemp[1])
                        if err > 0:
                            avals[pos] += val 
                            sum[pos]   += 1.0
                            sum2[pos]  += val * val
#
#--- open file for print out
#
        file = './Data/cti_data/ccd' + str(ccd) + '_data'
        fo   = open(file, 'w')
        line = '#\n#date       cti     errer\n#\n'
        fo.write(line)
        
        chk  = 0
        xvals = []
        yvals = []
        evals = []
        for k in range(2000, lyear+1):
            for m in range(0, 13):
                if (k == lyear) and (m > lmonth):
                    chk = 1
                    break

                pos = 12 * (k - 2000) + m -1
#
#--- set time in fractional year. adding 0.04 to set time to the mid month
#
                date = k + float(m) / 12.0 + 0.04     
                date = round(date, 3)
#
#--- compute average and erorr
#
                if avals[pos] > 0:
                    avg = avals[pos] / sum[pos]
                    err = math.sqrt(sum2[pos] /sum[pos] - avg * avg)
                    avg =  round(avg, 3)
                    err =  round(err, 3)

                    if len(str(date)) == 7:
                        line = str(date) + ' \t' + str(avg) + '\t' + str(err) + '\n'
                    else:
                        line = str(date) + '\t'  + str(avg) + '\t' + str(err) + '\n'
                    fo.write(line)


                    xvals.append(date)
                    yvals.append(avg)
                    evals.append(err)


            if chk  > 0:
                break

        fo.close()
#
#--- create a lists of lists
#
        xSets.append(xvals)
        ySets.append(yvals)
        eSets.append(evals)

    return [xSets, ySets, eSets]
def write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos, asig1, asig2, asig3, \
               mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):
    """
    write a html page:
    input: ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos,  max_acc,max_apos,m10_acc
           m10_apos,mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos
    """

    (lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday,
     dst) = tcnv.currentTime('Local')

    f = open('acis.html', 'w')

    #
    #--- this is a html 5 document
    #
    f.write('<!DOCTYPE html>\n')
    f.write('<html>\n')
    f.write('<head>\n')
    line = '<title>ACIS ' + ccd.upper() + ' Section ' + str(
        sec) + ' History Data</title>\n'
    f.write(line)
    f.write(
        "<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />"
    )
    #
    #--- css style sheet
    #
    f.write('<style type="text/css">\n')
    f.write(
        'table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n'
    )
    f.write('td {text-align:center}\n')
    f.write('a:link {color:aqua;}\n')
    f.write('a:visited {color:green;}\n')
    f.write('</style>\n')

    f.write('</head>\n')

    f.write('<body style="color:#FFFFFF;background-color:black"> \n')

    line = '<br /><h3> Last Update: ' + str(lmon) + '/' + str(
        lday) + ' / ' + str(lyear) + '</h3>\n'
    f.write(line)
    f.write('<table border=1>\n')

    header_write(f)

    for i in range(0, len(year)):
        f.write('<tr>\n')
        line = '<td>' + str(year[i]) + '</td>\t'
        line = line + '<td>' + str(month[i]) + '</td>\t'
        line = line + '<td>' + str(mean_dff[i]) + '</td>\t'
        line = line + '<td>' + str(std_dff[i]) + '</td>\t'
        line = line + '<td>' + str(min_dff[i]) + '</td>\t'
        line = line + '<td>' + str(min_dpos[i]) + '</td>\t'
        line = line + '<td>' + str(max_dff[i]) + '</td>\t'
        line = line + '<td>' + str(max_dpos[i]) + '</td>\t'
        line = line + '<td>' + str(dsig1[i]) + '</td>\t'
        line = line + '<td>' + str(dsig2[i]) + '</td>\t'
        line = line + '<td>' + str(dsig3[i]) + '</td>\t'
        f.write(line)
        syear = str(year[i])
        smon = str(month[i])
        if month[i] < 10:
            smon = '0' + smon
        lccd = ccd.replace('_', '')
        file = 'ACIS_' + smon + '_' + syear + '_' + lccd

        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/' + file + '.fits.gz">fits</a></td>\n'
        f.write(line)
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td>\n\n'
        f.write(line)

        line = '<td>' + str(mean_acc[i]) + '</td>\t'
        line = line + '<td>' + str(std_acc[i]) + '</td>\t'
        line = line + '<td>' + str(min_acc[i]) + '</td>\t'
        line = line + '<td>' + str(min_apos[i]) + '</td>\t'
        line = line + '<td>' + str(max_acc[i]) + '</td>\t'
        line = line + '<td>' + str(max_apos[i]) + '</td>\t'
        line = line + '<td>' + str(asig1[i]) + '</td>\t'
        line = line + '<td>' + str(asig2[i]) + '</td>\t'
        line = line + '<td>' + str(asig3[i]) + '</td>\n'
        f.write(line)

        file = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/' + file + '.fits.gz">fits</a></td>\n'
        f.write(line)
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td></tr>\n\n'
        f.write(line)
        #
        #--- put header every new year so that we can read data easier
        #
        if month[i] % 12 == 0 and i != (len(year) - 1):
            header_write(f)
            f.write('\n')

    f.write('</table>\n\n')
    f.write('<br /><br /><hr /><br />\n')

    line = '<br /><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + str(
        lday) + '/' + str(lyear) + '</strong>\n'
    f.write(line)

    line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
    f.write(line)
    f.write('</body>\n')
    f.write('</html>\n')

    f.close()
Example #36
0
def update_html():
    """
    create/update error html page for a gvien machine and a user
    input: error_list_<cpu>_<user>, wheren cpu and user are found from the machine running this and account using.
    output: cron_error_<cpu>_<usr>.html in html_dir.
    """

    #
    #--- find the current time
    #
    [year, mon, day, hours, min, sec, weekday, yday,
     dst] = tcnv.currentTime('Local')

    syear = str(year)
    smon = str(mon)
    lmon = tcnv.changeMonthFormat(mon)

    if mon < 10:
        smon = '0' + smon
#
#--- set the file name and a hmtl page name
#
    file = error_log_dir + machine + '_' + user + '_' + smon + '_' + syear
    html = html_dir + 'cron_error_' + machine + '_' + user + '_' + smon + '_' + syear + '.html'
    #
    #--- start writing the html page
    #
    out = open(html, 'w')
    line = '<!DOCTYPE html>\n'
    line = line + '<html>\n'
    line = line + '<head>\n'
    line = line + '<title>Cron Error Log for ' + user.upper(
    ) + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</title>\n'
    line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
    line = line + '</head>\n'
    line = line + '<body>\n'
    line = line + '<h3 style="padding-bottom: 10px"> Cron Error Log for ' + user.upper(
    ) + ' on ' + machine.upper() + ': ' + lmon + ' ' + syear + '</h3>\n\n'
    line = line + '<hr />\n'
    line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'

    out.write(line)
    #
    #--- write the content of error_list
    #
    f = open(file, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    for ent in data:
        out.write(ent)
        out.write('\n')

    line = '</pre>\n'
    line = line + '<br /><hr /><br />\n'
    line = line + 'Back to <a href="https://cxc.cfa.harvard.edu/mta_days/mta_cron/cron_error_main.html">Top Page</a>\n'
    line = line + '\n</body>\n'
    line = line + '</html>\n'
    out.write(line)

    out.close()
Example #37
0
def update_html():
    """
    update the main html page (sib_main.html)
    Input: none, but read a part from <house_keeping>/sim_head_part
    Output: <web_dir>/sib_main.html
    """
    #
    #--- find today's date, and set a few thing needed to set output directory and file name
    #
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    syear = str(year)
    smonth = str(mon)
    if mon < 10:
        smonth = '0' + smonth

    lyear = year
    lmon = mon - 1
    if lmon < 1:
        lmon = 12
        lyear -= 1

    slyear = str(lyear)
    slmonth = str(lmon)
    if lmon < 10:
        slmonth = '0' + slmonth
#
#--- read the head part from the house_keeping
#
    if level == 1:
        cmd = 'cp ' + house_keeping + '/sib_head_part ' + web_dir + '/sib_main.html'
    else:
        cmd = 'cp ' + house_keeping + '/sib_head_part2 ' + web_dir + '/sib_main_lev2.html'
    os.system(cmd)
    #
    #--- add the rest
    #
    if level == 1:
        line = web_dir + '/sib_main.html'
    else:
        line = web_dir + '/sib_main_lev2.html'
    fo = open(line, 'a')
    for iyear in range(year, 1999, -1):

        fo.write("<tr>\n")

        if iyear == year:
            line = '<th>' + syear + '</th><td>---</td>'
        else:
            if level == 1:
                line = '<th>' + str(
                    iyear) + '</th><td><a href="./Plots/Plot_' + str(
                        iyear) + '/year.html">' + str(iyear) + '</a></td>'
            else:
                line = '<th>' + str(
                    iyear) + '</th><td><a href="./Lev2/Plots/Plot_' + str(
                        iyear) + '/year.html">' + str(iyear) + '</a></td>'
        fo.write(line)
        fo.write("\n")
        for imonth in range(1, 13):
            simonth = str(imonth)
            if imonth < 10:
                simonth = '0' + simonth

            if (iyear == year) and (imonth >= mon):
                line = '<td>' + simonth + '</td>'
            else:
                if level == 1:
                    line = '<td><a href="./Plots/Plot_' + str(
                        iyear
                    ) + '_' + simonth + '/month.html">' + simonth + '</a></td>'
                else:
                    line = '<td><a href="./Lev2/Plots/Plot_' + str(
                        iyear
                    ) + '_' + simonth + '/month.html">' + simonth + '</a></td>'
            fo.write(line)
            fo.write("\n")

        fo.write("</tr>\n")

    fo.write("</table>\n")

    line = '<p style="padding-top:40px;padding-bottom:20px"> \n <hr /> \n </p> \n'
    fo.write(line)
    #
    #--- add updated date
    #
    date = tcnv.currentTime(format='Display')

    line = '<p style="padding-top:10px">Last Updated: ' + str(
        date) + '<br />' + "\n"
    fo.write(line)
    line = '<em style="padding-top:10px">If you have any questions, contact: <a href="mailto:[email protected]">[email protected]</a></p>'
    fo.write(line)
    fo.write("\n")
    fo.write("</body>\n")
    fo.write("</html>\n")

    fo.close()
Example #38
0
def ccd_comb_plot(choice, syear = 2000, smonth = 1, eyear = 2000, emonth = 1, header = 'plot_ccd'):

    """
    a control script to create plots
    Input:  choice      --- if normal, monthly updates of plots are created.
                            if check,  plots for a given period are created
            syear       --- starting year of the period,  choice must be 'check'
            smonth      --- starting month of the period, choice must be 'check'
            eyear       --- ending year of the period,    choice must be 'check'
            emonth      --- ending month of the period,   choice must be 'check'
            header      --- a header of the plot file     choice must be 'check'
    Output: png formated plotting files
    """
#
#--- find today's date, and set a few thing needed to set output directory and file name
#
    if choice != 'check':
        [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

        syear  = str(year)
        smonth = str(mon)
        if mon < 10:
            smonth = '0'+ smonth

        lyear = year 
        lmon  = mon - 1
        if lmon < 1:
            lmon   = 12
            lyear -= 1
    
        slyear  = str(lyear)
        slmonth = str(lmon)
        if lmon < 10:
            slmonth = '0' + slmonth
#
#--- normal monthly operation
#
    if choice == 'normal':
#
#--- monthly plot
#
        dlist    = collect_data_file_names('month')
        plot_out = web_dir + '/Plots/Plot_' +  slyear + '_' + slmonth + '/'
        check_and_create_dir(plot_out)
        header   =  'month_plot_ccd'
        plot_data(dlist, plot_out, header, yr=slyear, mo=slmonth,  psize=2.5)
#
#--- quarterly plot
#
        dlist    = collect_data_file_names('quarter')
        plot_out = web_dir + '/Plots/Plot_quarter/'
        check_and_create_dir(plot_out)
        header   = 'quarter_plot_ccd'
        plot_data(dlist, plot_out, header)
    
        dlist    = collect_data_file_names('year')
        plot_out = web_dir + '/Plots/Plot_past_year/'
        check_and_create_dir(plot_out)
        header   = 'one_year_plot_ccd'
        plot_data(dlist, plot_out, header)
#
#--- full previous year's plot. only updated in Jan of new year
#
        if mon == 1:
            dlist    = collect_data_file_names('lyear')
            lyear    = year -1
            plot_out = web_dir + '/Plots/Plot_' + str(lyear) + '/'
            check_and_create_dir(plot_out)
            header   =  'year_plot_ccd'
            plot_data(dlist, plot_out, header, yr=slyear)
#
#--- entire trend plot
#
        dlist    = collect_data_file_names('full')
        plot_out = web_dir + '/Plots/Plot_long_term/'
        check_and_create_dir(plot_out)
        header   = 'full_plot_ccd'
        plot_data(dlist, plot_out, header, xunit='year')
#
#--- special case which we need to specify periods
#
    elif choice == 'check':
    
        dlist  = collect_data_file_names('check', syear, smonth, eyear, emonth)
        plot_out = web_dir + '/Plot/'
        check_and_create_dir(plot_out)
#        header   = 'plot_special_ccd'
        plot_data(dlist, plot_out, header, yr=syear, mo=smonth)
#
#--- extra...
#
    else:
        for year in range(2000, 2014):

            dlist  = collect_data_file_names('check', year, 1, year, 12)
            plot_out = web_dir + '/Plots/Plot_' + str(year) + '/'
            check_and_create_dir(plot_out)
            header = 'one_year_plot_ccd'
            plot_data(dlist, plot_out, header, yr = str(year))

            for month in range(1, 13):
                print " Processing: " + str(year) + ' / ' + str(month)
                smonth = str(month)
                if month < 10:
                    smonth = '0' + smonth
    
                dlist = collect_data_file_names('check', year, month, year, month)
                plot_out = web_dir + '/Plots/Plot_' + str(year) + '_' + smonth +  '/'
                check_and_create_dir(plot_out)
                header = 'month_plot_ccd'
                plot_data(dlist, plot_out, header, yr=str(year), mo=smonth, psize=2.5)
Example #39
0
def print_html(year, mon):
    """
    create and/or update radiation related html page
    """
    #
    #--- find today's date
    #
    if year == '':
        [year, mon, day, hours, min, sec, weekday, yday,
         dst] = tcnv.currentTime()
#
#--- find out the last month
#
#        cyear = year
#        lmon      = mon -1
#        if lmon < 1:
#            lmon  = 12
#            cyear = year -1

#
#--- for the case year and mon are given
#
    cyear = year
    lmon = mon
    #
    #--- choose a correct month list depending on whether this is the leap year
    #
    if tcnv.isLeapYear(cyear) == 1:
        mon_list = mon_list1
    else:
        mon_list = mon_list2

    last_day = mon_list[lmon - 1]
    #
    #--- convert the month from a numeric to letter
    #
    umon = tcnv.changeMonthFormat(lmon)
    smon = umon.lower()

    lmon_year = str(cyear)
    syear = lmon_year[2] + lmon_year[3]
    last_year = str(year - 1)
    syear2 = last_year[2] + last_year[3]
    monyear = smon + syear
    #
    #--- set output html page names
    #
    year_html = 'all' + syear + '.html'
    mon_html = monyear + '.html'
    rad_html = 'rad_time_' + monyear + '.html'
    #
    #--- read yearly html page template
    #
    data = open('./Template/yearly_template', 'r').read()

    data = data.replace('$#FYEAR#$', str(year))
    data = data.replace('$#SYEAR#$', syear)

    fo = open(year_html, 'w')
    fo.write(data)
    fo.close()
    #
    #--- read monthly html page template
    #
    data = open('./Template/monthly_template', 'r').read()

    data = data.replace('$#FYEAR#$', str(year))
    data = data.replace('$#UMON#$', umon)
    data = data.replace('$#MONYEAR#$', monyear)

    fo = open(mon_html, 'w')
    fo.write(data)
    fo.close()
    #
    #--- read rad_time html page template
    #
    data = open('./Template/rad_time_template', 'r').read()

    data = data.replace('$#LMONTH#$', fmon_list[mon - 2])
    data = data.replace('$#FYEAR#$', str(year))
    data = data.replace('$#MONYEAR#$', monyear)

    fo = open(rad_html, 'w')
    fo.write(data)
    fo.close()
Example #40
0
def plot_cum_grating(time, h_in, l_in):
    """
    plot cummulative count rates of hetig and letig insertion. 
    input:  time    --- fractional year
            h_in    --- hetig insertion cummulative count rate (month step)
            l_in    --- letig insertion cummulative count rate 
    output: monthly_grat.prn
    """
    #
    #--- set x axis plotting range
    #
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
    xmin = 2000
    xmax = year + 1
    if mon > 6:
        xmax += 1
#
#--- set y axis plotting range
#
    ymin = 0.0
    ymax = max(h_in)
    ymax2 = max(l_in)
    if ymax2 > ymax:
        ymax = ymax2
    ymax = int(1.1 * ymax) + 10
    #
    #--- set a few parameters
    #
    fsize = 9
    lsize = 0
    color = 'red'
    marker = 'o'
    msize = 3
    plt.close("all")
    mpl.rcParams['font.size'] = fsize
    props = font_manager.FontProperties(size=fsize)
    plt.subplots_adjust(hspace=0.08, wspace=0.10)
    #
    #--- HETG Cumulative Count Plots
    #
    a1 = plt.subplot(121)  #--- two panel plot: left
    plot_sub(a1,
             time,
             h_in,
             xmin,
             xmax,
             ymin,
             ymax,
             color,
             lsize,
             marker,
             msize,
             tline='HETG')

    a1.set_xlabel('Time (year)', size=fsize)
    a1.set_ylabel('Cumulative Insertion Counts', size=fsize)
    #
    #--- LETG Cumulative Count Plots
    #
    a1 = plt.subplot(122)  #--- two panel plot: right
    plot_sub(a1,
             time,
             l_in,
             xmin,
             xmax,
             ymin,
             ymax,
             color,
             lsize,
             marker,
             msize,
             tline='LETG')

    a1.set_xlabel('Time (year)', size=fsize)
    #
    #--- save the plot
    #
    fig = matplotlib.pyplot.gcf()
    fig.set_size_inches(10.0, 7.5)
    outname = 'monthly_grat.png'
    plt.savefig(outname, format='png', dpi=100)
Example #41
0
def get_grat_data():
    """
    read database and extract needed information, then create data
    input: none but read from the database: "/data/mta/www/mta_otg/OTG_sorted.rdb"
    output: [time, h_in_ang, h_out_ang, l_in_ang, l_out_ang, h_in, h_out, l_in, l_out]
            where: time         --- time in fractional year
                   h_in_ang     --- hetig insertion angle
                   h_out_ang    --- hetig retraction angle
                   l_in_ang     --- letig insertion angle
                   l_out_ang    --- letig retraction angle
                   h_in         --- hetig insertion cumm count
                   h_out        --- hetig retraction cumm count
                   l_in         --- letig insertion cumm count
                   l_out        --- hetig retraction cumm count
    """
#
#--- read data
#
    f    = open(datafile, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
#
#--- find the current year. this will be used to remove iregular data
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()

    direct   = []
    grating  = []
    start    = []
    stop     = []
    hposa    = []
    hposb    = []
    fposa    = []
    fposb    = []

    for i in range(1, len(data)):
        ent   = data[i]
        atemp = re.split('\s+', ent)
        try:
            test  = float(atemp[2])
            test2 = float(atemp[4])
        except:
            continue
        direct.append(atemp[0].strip())
        grating.append(atemp[1].strip())
        val = convert_time(atemp[2])

        start.append(val)
        stop.append(convert_time(atemp[4]))
        hposa.append(float(atemp[18]))
        hposb.append(float(atemp[19]))
        fposa.append(float(atemp[20]))
        fposb.append(float(atemp[21]))
#
#--- create start and stop lists of data bin. the width is a month
#
    [blist, elist] = create_monthly_bins(2000, year, mon)

    blen          = len(blist)
    time          = [0 for x in range(0, blen)]
    h_in          = [0 for x in range(0, blen)]     #--- hetg insertion cumm count rate
    h_in_ang      = [0 for x in range(0, blen)]     #--- hetg insertion angle
    h_out         = [0 for x in range(0, blen)]     #--- hetg retraction cumm count rate
    h_out_ang     = [0 for x in range(0, blen)]     #--- hetg retraction angle

    l_in          = [0 for x in range(0, blen)]
    l_in_ang      = [0 for x in range(0, blen)]
    l_out         = [0 for x in range(0, blen)]
    l_out_ang     = [0 for x in range(0, blen)]

    for j in range(1, blen):

        time[j]       = 0.5 * (blist[j] + elist[j]) #--- take a mid point for the bin's time 
#
#-- creating cummulative count; the current bin should have, at least, as the same as the 
#-- previous bin
#
        h_in[j]       = h_in[j-1]
        h_out[j]      = h_out[j-1]
        l_in[j]       = l_in[j-1]
        l_out[j]      = l_out[j-1]

        h_in_ang_cnt  = 0
        h_out_ang_cnt = 0
        l_in_ang_cnt  = 0
        l_out_ang_cnt = 0
#
#--- since the data are not ordered by date, go through begining to the end 
#--- every bin cycle
#
        for i in range(0, len(start)):

            if start[i] >= blist[j] and start[i] < elist[j]: 
                if direct[i] == 'INSR':
                    if grating[i] == 'HETG':
                        h_in_ang[j]   += fposa[i]
                        h_in_ang_cnt  += 1
                        h_out_ang[j]  += hposa[i]
                        h_out_ang_cnt += 1
     
                    if grating[i] == 'LETG':
                        l_in_ang[j]   += fposa[i]
                        l_in_ang_cnt  += 1
                        l_out_ang[j]  += hposa[i]
                        l_out_ang_cnt += 1
#
#--- taking monthly average
#
        if h_in_ang_cnt> 0:
            h_in_ang[j] /=  h_in_ang_cnt

        if h_out_ang_cnt> 0:
            h_out_ang[j] /=  h_out_ang_cnt

        if l_in_ang_cnt> 0:
            l_in_ang[j] /=  l_in_ang_cnt

        if l_out_ang_cnt> 0:
            l_out_ang[j] /=  l_out_ang_cnt
#
#--- adding in/out count for the month to appropriate bins
#
        h_in[j]  += h_in_ang_cnt
        h_out[j] += h_out_ang_cnt
        l_in[j]  += l_in_ang_cnt
        l_out[j] += l_out_ang_cnt

    return [time, h_in_ang, h_out_ang, l_in_ang, l_out_ang, h_in, h_out, l_in, l_out]
Example #42
0
def clean_data(dir,
               startYear='NA',
               startMonth='NA',
               stopYear='NA',
               stopMonth='NA'):
    """
    clean up and correct ACIS/HRC data. if there is duplicated line, remove it. if there are missing line add one (with NA)
    input dir, startYear = 1999, startMonth = 9, stopYear, stopMonth
     
    """
    #
    #--- if range is not defined, give them
    #
    if startYear == 'NA':
        startYear = 1999
    if startMonth == 'NA':
        startMonth = 9

    (cyear, cmon, day, hours, min, sec, weekday, yday,
     dst) = tcnv.currentTime('Local')
    if stopYear == 'NA':
        stopYear = cyear
    if stopMonth == 'NA':
        stopMonth = cmon - 1
        if stopMonth < 1:
            stopMonth = 12
            stopYear -= 1

    for type in ('*_acc*', '*_dff*'):

        #
        #--- find file names
        #
        for fout in os.listdir(dir):
            if fnmatch.fnmatch(fout, type):
                ent = dir + fout

                f = open(ent, 'r')
                data = [line.strip() for line in f.readlines()]
                f.close()
                f = open('zout', 'w')

                cyear = startYear
                cmonth = startMonth
                pyear = 0
                pmonth = 0

                for aent in data:

                    atemp = re.split('\s+|\t+', aent)
                    year = int(atemp[0])
                    month = int(atemp[1])
                    #
                    #--- if entry is duplicated, remove
                    #
                    if year == pyear and month == pmonth:
                        pass
                    elif year == cyear and month == cmonth:
                        f.write(aent)
                        f.write('\n')
                        pyear = year
                        pmonth = month
                        cmonth += 1
                        if cmonth > 12:
                            cmonth = 1
                            cyear += 1
                            if cyear == stopYear and cmonth > stopMonth:
                                break
#
#--- if entries are missing, add "NA"
#
                    elif year == cyear and month > cmonth:
                        for i in range(cmonth, month):
                            smon = str(i)
                            if i < 10:
                                smon = '0' + smon
                            line = str(
                                year
                            ) + '\t' + smon + '\tNA      NA      NA      NA      NA      NA      NA      NA\n'
                            f.write(line)
                        f.write(aent)
                        f.write('\n')

                        pyear = year
                        pmonth = month
                        cmonth += 1
                        if cmonth > 12:
                            cmonth = 1
                            cyear += 1
                            if cyear == stopYear and cmonth > stopMonth:
                                break

                f.close()

                cmd = 'mv zout ' + ent
                os.system(cmd)
Example #43
0
def update_main_html():
    """
    update the main cron error log html page 
    input: get from the list from indivisula html page, e.g., cron_error_rhodes_mta.html
    output: cron_error_main.html
    """

    #
    #--- create a list of file name (header)
    #

    #    file_list = []
    #    for cpu in cpu_list:
    #        for name in usr_list:
    #            filename = cpu + '_' + name
    #            file_list.append(filename)

    file_list = cpu_usr_list  #-- we may go back to above scheme in future, but this si fine for now

    #
    #--- find current time
    #
    [year, mon, day, hours, min, sec, weekday, yday,
     dst] = tcnv.currentTime('Local')

    syear = str(year)
    smon = str(mon)
    lmon = tcnv.changeMonthFormat(mon)

    if mon < 10:
        smon = '0' + smon

#
#--- start writing the html page
#
    html = html_dir + 'cron_error_main.html'

    out = open(html, 'w')
    line = '<!DOCTYPE html>\n'
    line = line + '<html>\n'
    line = line + '<head>\n'
    line = line + '<title>Cron Error Main page</title>\n'
    line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
    line = line + '</head>\n'
    line = line + '<body>\n'
    line = line + '<h2 style="padding-bottom: 10px"> Cron Error Log</h2>\n\n'
    line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n'
    line = line + '<hr />\n'
    line = line + '<table border=2 cellpadding = 5 cellspacing =5>\n'
    line = line + '<tr><th>Period</th>'
    out.write(line)

    for ent in file_list:
        atemp = re.split('_', ent)
        line = '<th>' + atemp[1] + ' on ' + atemp[0] + '</th>'
        out.write(line)

    out.write('</tr>\n')
    #
    #--- find the names of each html file (e.g. cron_error_rhodes_mta_06_2012.html)
    #
    templist = tempdir + 'zlist'
    cmd = 'ls ' + html_dir + '> ' + templist
    os.system(cmd)
    f = open(templist, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    cmd = 'rm ' + templist
    os.system(cmd)

    error_file_list = []

    for ent in data:
        m1 = re.search('.html', ent)
        m2 = re.search('cron_error_main.html', ent)
        if (m1 is not None) and (m2 is None):
            error_file_list.append(ent)

#
#--- start printing each row; column is ordered newest to oldeest
#
    year_list = range(2012, year + 1)
    year_list.reverse()
    month_list = range(1, 13)
    month_list.reverse()

    for dyear in year_list:
        for dmonth in month_list:
            if dyear == 2012 and dmonth < 6:  #---- this is the year/month the script was started
                break

            if (dyear < year) or (dyear == year and dmonth <= mon):

                lmon = tcnv.changeMonthFormat(
                    dmonth)  #--- convert month in digit to letters

                line = '<tr><th>' + lmon + ' ' + str(dyear) + '</th>'
                out.write(line)
                #
                #--- check which file (e.g. cron_error_rhodes_mta_06_2012.html) actually exists
                #
                for fent in file_list:
                    smon = str(dmonth)
                    if dmonth < 10:
                        smon = '0' + smon
                    fname = 'cron_error_' + fent + '_' + smon + '_' + str(
                        dyear) + '.html'
                    chk = 0
                    for comp in error_file_list:
                        if fname == comp:
                            chk = 1
                            break
                    if chk > 0:
                        #
                        #--- if exist, create a link
                        #
                        line = '<td style="color:red;text-align:center"><a href="' + fname + '">Error List</a></td>'
                    else:
                        line = '<td style="text-align:center">No Error</td>'
                    out.write(line)
                out.write('</tr>\n')

    out.write('</table>\n\n')

    out.write('<br /> <hr />\n')

    tdate = current_time_from_machine()

    line = '<pstyle="font-size:95%"><em>Last Update: ' + tdate + '</em><br />\n'
    line = line + 'If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected]</a>.</p>\n'
    out.write(line)

    line = '\n</body>\n'
    line = line + '</html>\n'
    out.write(line)

    out.close()
def acis_dose_make_data_html(indir='NA', outdir='NA', comp_test='NA'):
    """
    read data and create html pages
    input: indir, outdir, both are optional
    """
    #
    #--- setting indir and outdir if not given
    #
    if indir == 'NA':
        if comp_test == 'test':
            indir = test_data_out
        else:
            indir = data_out

    if outdir == 'NA':
        if comp_test == 'test':
            outdir = test_data_out
        else:
            outdir = data_out

#
#--- read data
#
    for ccd in ('i_2', 'i_3', 's_2', 's_3'):
        for sec in range(0, 4):

            inst = ccd + '_n_' + str(sec)

            date = []
            year = []
            month = []
            mean_acc = []
            std_acc = []
            min_acc = []
            min_apos = []
            max_acc = []
            max_apos = []
            asig1 = []
            asig2 = []
            asig3 = []
            mean_dff = []
            std_dff = []
            min_dff = []
            min_dpos = []
            max_dff = []
            max_dpos = []
            dsig1 = []
            dsig2 = []
            dsig3 = []

            [date,year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff, \
              min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = expf.readExpData(indir, inst)

            #
            #--- write html page
            #
            write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, \
                                 mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3)

            out_name = outdir + inst + '.html'
            cmd = 'mv acis.html ' + out_name
            os.system(cmd)
#
#--- update top html page
#
    line = house_keeping + 'exposure.html'
    f = open(line, 'r')
    data = f.read()
    f.close()

    now = tcnv.currentTime("Display")
    ndate = 'Last Update: ' + now
    ndata = data.replace("Last Update:", ndate)

    line = web_dir + 'exposure.html'
    f = open(line, 'w')
    f.write(ndata)
    f.close()
    #
    #--- update plot page htmls
    #
    update_plt_html_date(comp_test)
def sci_run_get_radiation_data():

    'extract needed radiation data from /data/mta4/www/DAILY/mta_rad/ACE/, and put in rada_data<YYYYY>, where YYYY is the year'
    #
    #--- find out today's date in Local time frame
    #

    if comp_test == 'test':
        year = 2012
        month = 1
        day = 2
    else:
        today = tcnv.currentTime('local')
        year = today[0]
        month = today[1]
        day = today[2]

    if month == 1 and day == 1:
        #
        #--- this is a new year... complete the last year
        #
        year -= 1

#
#--- extract data form ACE data files
#

    line = '/data/mta4/www/DAILY/mta_rad/ACE/' + str(
        year) + '*_ace_epam_5m.txt'
    cmd = 'cat ' + line + ' > /tmp/mta/ztemp'
    cmd = 'cat ' + line + ' > ./ztemp'
    os.system(cmd)

    #>    f = open('/tmp/mta/ztemp', 'r')
    f = open('./ztemp', 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    #>    system('rm /tmp/mta/ztemp')
    os.system('rm ./ztemp')

    #
    #--- move the old file to "~" to prepare for the new data
    #

    name = 'rad_data' + str(year)

    if comp_test == 'test':
        crtFile = test_web_dir + name
    else:
        oldFile = house_keeping + name + '~'
        crtFile = house_keeping + name

        cmd = 'chmod 775 ' + crtFile + ' ' + oldFile
        os.system(cmd)

        cmd = 'mv ' + crtFile + ' ' + oldFile
        os.system(cmd)

        cmd = 'chmod 644 ' + oldFile
        os.system(cmd)

    f = open(crtFile, 'w')

    for ent in data:
        #
        #--- remove comments and headers
        #
        m = re.search('^#', str(ent))
        n = re.search('^:', str(ent))
        if (m is None) and (n is None):
            line = ent + '\n'
            f.write(line)

    f.close()
Example #46
0
def check_cron():
    """
    find new error messages from cron log files for a given machine and a given user.
    this script send out email if it finds new error message. The log files are clean up 1 of every month
    and the same message could be send out if there are still the same error messages occur.
    """
    #
    #--- error_log name
    #
    error_logs = error_log_dir + 'error_list_' + machine + '_' + user

    #
    #--- find cron file names for this machine for this user
    #
    cron_file_name = extract_cron_file_name()

    #
    #--- check today's date. if it is 1st of the month, move the old error_list to archive form.
    #

    [year, mon, day, hours, min, sec, weekday, yday,
     dst] = tcnv.currentTime('Local')

    if day == 1:
        lyear = year
        lmon = mon - 1
        if lmon < 1:
            lmon = 12
            lyear -= 1

        error_logs_old = error_logs + '_' + str(lmon) + '_' + str(lyear)
        cmd = 'mv ' + error_logs + ' ' + error_logs_old
        os.system(cmd)
        cmd = 'mv ' + error_logs_old + ' ' + error_log_dir + 'Past_logs/.'
        os.system(cmd)

        error_dict = {}
    else:
        #
        #--- read existing error list
        #
        error_dict = {}
        try:
            f = open(error_logs, 'r')
            data = [line.strip() for line in f.readlines()]
            f.close()
            for ent in data:
                atemp = re.split('\<:\>', ent)
                content = []
                for i in range(1, len(atemp)):
                    content.append(atemp[i])

                error_dict[atemp[0]] = content
        except:
            pass

#
#--- set which Log location to check (depends on user.)
#
    dir_loc = '/home/' + user + '/Logs/'

    #
    #--- find names of files and directories in the Logs directory
    #
    cmd = 'ls -lrtd ' + dir_loc + '/* >' + tempout
    os.system(cmd)
    f = open(tempout, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    cmd = 'rm ' + tempout
    os.system(cmd)

    cron_list = []
    for ent in data:
        atemp = re.split('\s+|\t+', ent)
        m1 = re.search('d', atemp[0])
        m2 = re.search('Past_logs', ent)
        #
        #--- if it is a directory other than Past_logs, find file names in that directory
        #
        if (m1 is not None) and (m2 is None):
            cmd = 'ls ' + atemp[8] + '/* > ' + tempout
            os.system(cmd)
            f = open(tempout, 'r')
            data2 = [line.strip() for line in f.readlines()]
            f.close()
            cmd = 'rm ' + tempout
            os.system(cmd)

            for ent2 in data2:
                cron_list.append(ent2)
#
#--- files in Logs directory level
#
        elif m2 is None:
            cron_list.append(atemp[8])

    new_error_dict = {}
    for file in cron_list:
        #
        #--- check whether this error message belongs to this machine (and the user)
        #
        mchk = 0
        for comp in cron_file_name:
            m = re.search(comp, file)

            if m is not None:
                mchk = 1

        if mchk > 0:
            #
            #--- check whether the file has any error messages
            #
            error_list = find_error(file)
            if len(error_list) > 0:
                #
                #--- if there are error messages, compare them to the previous record, and if it is new append to the record.
                #
                try:
                    prev_list = error_dict[file]
                    new_error = []
                    for ent in error_list:
                        sent = "".join(
                            ent.split())  #---- removing all white spaces
                        chk = 0
                        for comp in prev_list:
                            scomp = "".join(comp.split())
                            if sent == scomp:
                                chk = 1

                        if chk == 0:
                            prev_list.append(ent)
                            new_error.append(ent)

                    if len(new_error) > 0:
                        error_dict[file] = prev_list
                        new_error_dict[file] = new_error
                except:
                    #
                    #--- there is no previous error_list entry: so all error messages are new and log them
                    #
                    error_dict[file] = error_list
                    new_error_dict[file] = error_list

#
#--- update error logs
#
    old_log = error_logs + '~'
    cmd = 'mv ' + error_logs + ' ' + old_log
    os.system(cmd)

    f = open(error_logs, 'w')
    for key in error_dict:
        line = key
        for e_ent in error_dict[key]:
            line = line + '<:>' + e_ent
        line = line + '\n'
        f.write(line)

    f.close()

    #
    #---if new error messages are found; notify to a list of users
    #
    chk = 0
    f = open(tempout, 'w')
    for key in new_error_dict:
        chk += 1
        line = key + '\n'
        f.write(line)
        for ent in new_error_dict[key]:
            line = '\t' + ent + '\n'
            f.write(line)

        f.write('\n')

    f.close()

    if chk > 0:
        cmd = 'cat ' + tempout + ' | mailx -s "Subject: Cron Error : ' + user + ' on ' + machine + '"  ' + email_list
        os.system(cmd)
        #
        #--- add the error message to a recored
        #
        add_to_log()
        #
        #--- update html page
        #
        update_html()
        update_main_html()

    else:
        #
        #--- if there is no error, notify that fact to admin
        #
        f = open(tempout, 'w')
        line = '\nNo error is found today on ' + machine + ' by a user ' + user + '.\n'
        f.write(line)
        f.close()
        cmd = 'cat ' + tempout + ' | mailx -s "Subject: No Cron Error : ' + user + ' on ' + machine + '" ' + admin
        os.system(cmd)

    cmd = 'rm ' + tempout
    os.system(cmd)
Example #47
0
def  print_main_head_part(fo, new_bad_pix_list, new_hot_pix_list, new_col_pix_list):

    localt = tcnv.currentTime('UTC')
    tyear  = int(localt[0])
    tyday  = int(localt[7])
    dom    = tcnv.YdateToDOM(tyear, tyday)

    dispt  = tcnv.currentTime('Display')
    atemp  = re.split('\s+', dispt)
    dtime  = atemp[0] + ' ' +  atemp[1] + ' ' + atemp[2] +', ' + atemp[4]

    hfile  = house_keeping + 'head'
    head_part = open(hfile, 'r').read()
    head_part = head_part.replace('#DISPLAY# ', dtime)
    head_part = head_part.replace('#YDATE#', str(tyday))
    head_part = head_part.replace('#DOM#', str(dom))

    line = ''
    chk  = 0
    for i in range(0, 10):
        if new_bad_pix_list[i][0] != '':
            chk += 1
            line = line + 'CCD' + str(i) + ': '
            for ent in new_bad_pix_list[i]:
                line = line + ent
            line = line + '<br />\n'
    if chk == 0:
            line = 'No New Warm Pixel'

    head_part = head_part.replace('#WARMPIX#', line)


    line = ''
    chk  = 0
    for i in range(0, 10):
        if new_hot_pix_list[i][0] != '':
            chk += 1
            line = line + 'CCD' + str(i) + ': '
            for ent in new_hot_pix_list[i]:
                line = line + ent
            line = line + '<br />\n'
    if chk == 0:
            line = 'No New Hot Pixel'

    head_part = head_part.replace('#HOTPIX#', line)


    line = ''
    chk  = 0
    for i in range(0, 10):
        if new_col_pix_list[i][0] != '':
            chk += 1
            line = line + 'CCD' + str(i) + ': '
            for ent in new_col_pix_list[i]:
                line = line + ent
            line = line + '<br />\n'
    if chk == 0:
            line = 'No New Warm Column'

    head_part = head_part.replace('#WARMCOL#', line)

    fo.write(head_part)
Example #48
0
def print_index_html():
    """
    create and/or update radiation related html page
    """
    #
    #--- find today's date
    #
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime()
    #
    #--- read header part and the first part of the index page
    #
    line = open('./Template/index_top', 'r').read()
    #
    #--- start creating a link table
    #
    line = line + '<table border=1 cellpadding=10 cellspacing=2>\n'
    line = line + '<tr>\n'
    line = line + '<td colspan=13>\n'
    line = line + '<table border=1 width=100%>\n'
    line = line + '<tr><th> <a href="all.html" style="font-size:120%">Mission since JAN2010</a></th></tr>\n'
    line = line + '</table>\n'
    line = line + '</td>\n'
    line = line + '</tr>\n'

    line = line + '<tr>'
    line = line + '<th>Year</th><th>Jan</th><th>Feb</th><th>Mar</th><th>Apr</th><th>May</th><th>Jun</th>\n'
    line = line + '<th>Jul</th><th>Aug</th><th>Sep</th><th>Oct</th><th>Nov</th><th>Dec</th>\n'
    line = line + '</tr>\n'
    for eyear in range(year, 1999, -1):
        line = line + '<tr>\n'
        line = line + '<th>' + str(eyear) + '</th>\n'

        lyear = str(eyear)
        syear = lyear[2] + lyear[3]

        for emon in range(1, 13):
            lmon = tcnv.changeMonthFormat(emon)
            monyear = lmon.lower() + syear
            cmonyear = monyear.upper()

            if eyear == year and emon > mon:
                line = line + '<td>' + cmonyear + '</td>\n'
            else:
                line = line + '<td><a href="./' + monyear + '.html">' + cmonyear + '</a></td>\n'

        line = line + '</tr>\n\n'

    line = line + '</table>\n'
    #
    #--- table finished. add a closing part
    #
    line = line + '<div style="padding-top: 15px"></div>\n'
    line = line + '<hr />'
    line = line + '<div style="padding-top: 15px"></div>\n'
    line = line + '<p>This page is maintained by B. Spitzbart (<a href="*****@*****.**">[email protected]</a>).\n'
    line = line + '</body</html>\n'
    #
    #--- now write out the page
    #
    fo = open('/data/mta4/www/DAILY/mta_rad/index.html', 'w')
    fo.write(line)
    fo.close()
def update_main_html():

    """
    update the main cron error log html page 
    input: get from the list from indivisula html page, e.g., cron_error_rhodes_mta.html
    output: cron_error_main.html
    """

#
#--- create a list of file name (header)
#

#    file_list = []
#    for cpu in cpu_list:
#        for name in usr_list:
#            filename = cpu + '_' + name
#            file_list.append(filename)

    file_list = cpu_usr_list                    #-- we may go back to above scheme in future, but this si fine for now

#
#--- find current time
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('Local')

    syear = str(year)
    smon  = str(mon)
    lmon = tcnv.changeMonthFormat(mon)

    if mon < 10:
        smon = '0' + smon

#
#--- start writing the html page
#
    html = html_dir +  'cron_error_main.html'

    out  = open(html, 'w')
    line = '<!DOCTYPE html>\n'
    line = line + '<html>\n'
    line = line + '<head>\n'
    line = line + '<title>Cron Error Main page</title>\n'
    line = line + '<link rel="stylesheet" type="text/css" href="/mta/REPORTS/Template/mta_style_short.css" />\n'
    line = line + '</head>\n'
    line = line + '<body>\n'
    line = line + '<h2 style="padding-bottom: 10px"> Cron Error Log</h2>\n\n'
    line = line + '<pre style="padding-left: 5px;padding-bottom:10px">\n' 
    line = line + '<hr />\n'
    line = line + '<table border=2 cellpadding = 5 cellspacing =5>\n'
    line = line + '<tr><th>Period</th>'
    out.write(line)

    for ent in file_list:
        atemp = re.split('_', ent)
        line = '<th>' + atemp[1] + ' on ' + atemp[0]  + '</th>'
        out.write(line)

    out.write('</tr>\n')
#
#--- find the names of each html file (e.g. cron_error_rhodes_mta_06_2012.html)
#
    templist = tempdir + 'zlist'
    cmd = 'ls ' + html_dir + '> ' + templist
    os.system(cmd)
    f    = open(templist, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    cmd = 'rm ' + templist
    os.system(cmd)

    error_file_list = []

    for ent in data:
        m1 = re.search('.html', ent)
        m2 = re.search('cron_error_main.html', ent)
        if (m1 is not None) and (m2 is None):
            error_file_list.append(ent)

#
#--- start printing each row; column is ordered newest to oldeest
#
    year_list = range(2012, year + 1)
    year_list.reverse()
    month_list = range(1,13)
    month_list.reverse()

    for dyear in year_list:
        for dmonth in month_list:
            if dyear == 2012 and dmonth < 6:                    #---- this is the year/month the script was started
                break

            if (dyear < year) or (dyear == year  and dmonth <=  mon):

                lmon = tcnv.changeMonthFormat(dmonth)           #--- convert month in digit to letters

                line = '<tr><th>' + lmon + ' ' + str(dyear) + '</th>'
                out.write(line)
#
#--- check which file (e.g. cron_error_rhodes_mta_06_2012.html) actually exists
#
                for fent in file_list:
                    smon  = str(dmonth)
                    if dmonth < 10:
                        smon = '0' + smon
                    fname = 'cron_error_' + fent + '_' + smon + '_' + str(dyear) + '.html'
                    chk = 0
                    for comp in error_file_list:
                        if fname == comp:
                            chk = 1
                            break
                    if chk > 0:
#
#--- if exist, create a link
#
                        line = '<td style="color:red;text-align:center"><a href="' + fname + '">Error List</a></td>'
                    else:
                        line = '<td style="text-align:center">No Error</td>'
                    out.write(line)
                out.write('</tr>\n')

    out.write('</table>\n\n')

    out.write('<br /> <hr />\n')

    tdate = current_time_from_machine()

    line = '<pstyle="font-size:95%"><em>Last Update: ' + tdate + '</em><br />\n'
    line = line + 'If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected]</a>.</p>\n'
    out.write(line)

    line = '\n</body>\n'
    line = line + '</html>\n'
    out.write(line)

    out.close()
Example #50
0
    year = 2012
    month = 4
    day = 22
    #
    #--- create test directories
    #
    asrf.prep_for_test(web_dir, comp_test)

elif comp_test == 'test2':  #---- this is another test case, but go over two years 2011 and 2012
    year = 2012
    month = 2
    day = 2

    asrf.prep_for_test(web_dir, comp_test)
else:
    today = tcnv.currentTime('local')
    year = today[0]
    month = today[1]
    day = today[2]
#
#--- setting the current output directory
#
current_dir = 'Year' + str(year) + '/'

#-----------------------------------------------------------------------------------------------
#---acis_sci_run_get_data: extracts mit data and updates acis science run data and plots     ---
#-----------------------------------------------------------------------------------------------


def acis_sci_run_get_data():
    """
Example #51
0
def extractLimit(datloc):
    """
    read average and std values of each msid, compute lower and upper limits, and create limit data table
    Input: datloc: the name of the data directory under data_dir
    Output: data_dir/os_limit_table
    """
    #
    #--- read data set names from data directory
    #
    datpass = data_dir + datloc + '/'
    cmd = 'ls ' + datpass + '* >' + ztemp
    os.system(cmd)

    f = open(ztemp, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    cmd = 'rm ' + ztemp
    #    os.system(cmd)
    #
    #--- print header
    #
    outfile = data_dir + 'os_limit_table'
    fw = open(outfile, 'w')
    dtime = tcnv.currentTime('Display')
    line = '#Last Update: ' + dtime + '\n'
    fw.write(line)
    fw.write('#\t\t\tLower Limit\t\tUpper Limit\t\tGroup\n')
    fw.write('#msid\t\t Red\tYellow\tYellow\tRed\n')
    fw.write('#-------------------------------------------------------\n')

    #
    #---- start printing limit values
    #
    for ent in data:

        print ent
        temp = re.split(datpass, ent)
        group = temp[1]  #---- group ID

        if ent == outfile:  #---os_limt_table is in the data directory, too; so ignore it
            pass
        else:
            f = open(ent, 'r')
            msdat = [line.strip() for line in f.readlines()]
            f.close()

            colEnt = re.split('\s+|\t+', msdat[0])
            colLen = len(colEnt)
            colName = []
            #
            #--- read msid names
            #
            for ent in colEnt:
                if ent != 'std' and ent != '#time':
                    colName.append(ent)

#
#--- read average and std of each msid
#
            lastp = int(colLen / 2)
            for k in range(0, lastp + 1):
                exec("avg%d = []" % (k))
                exec("sig%d = []" % (k))

            total = 0
            for line in msdat:
                if total == 0:
                    total += 1
                else:
                    atemp = re.split('\s+|\t+', line)

                    for k in range(1, colLen):
                        k2 = int(k / 2)
                        if k % 2 == 0:
                            k2 -= 1
                            #
                            #--- the data is occasionally na; if that is the case, use the previous value. if it is the first one, set it to "0"
                            #
                            try:
                                exec("sig%d.append(float(atemp[%d]))" %
                                     (k2, k))
                            except:
                                try:
                                    exec("sig%d.append(sig%d[len(sig%d) -1])" %
                                         (k2, k2, k2))
                                except:
                                    exec("sig%d.append(0)" % (k2))
                        else:
                            try:
                                exec("avg%d.append(float(atemp[%d]))" %
                                     (k2, k))
                            except:
                                try:
                                    exec("avg%d.append(avg%d[len(avg%d) -1])" %
                                         (k2, k2, k2))
                                except:
                                    exec("avg%d.append(0)" % (k2))

                    total += 1

            total -= 2
            if total > 2:  #---- we need at least 4 data points
                fw.write('#\n')
                for k in range(0, len(colName)):
                    #
                    #--- compute limits; as for a deviation, use the average of the past two years
                    #
                    exec(
                        "madd = sig%d[%d] + sig%d[%d] + sig%d[%d] + sig%d[%d]"
                        % (k, total - 3, k, total - 2, k, total - 1, k, total))
                    madd /= 4.0
                    exec("lavg = avg%d[%d]" % (k, total))

                    upperYellow = lavg + 4.0 * madd
                    upperRed = lavg + 5.0 * madd
                    lowerYellow = lavg - 4.0 * madd
                    lowerRed = lavg - 5.0 * madd
                    if upperYellow == lowerYellow:
                        if lowerYellow == 0:
                            lowerYellow -= 0.01
                            upperYellow += 0.01
                            lowerRed -= 0.01
                            upperRed += 0.01
                        else:
                            diff = 0.01 * upperRed
                            lowerYellow -= diff
                            upperYellow += diff
                            lowerRed -= diff
                            upperRed += diff

                    if len(colName[k]) < 8:
                        line = '%s\t\t%5.3f\t%5.3f\t%5.3f\t%5.3f\t%s\n' % (
                            colName[k], lowerRed, lowerYellow, upperYellow,
                            upperRed, group)
                    else:
                        line = '%s\t%5.3f\t%5.3f\t%5.3f\t%5.3f\t%s\n' % (
                            colName[k], lowerRed, lowerYellow, upperYellow,
                            upperRed, group)

                    fw.write(line)

    fw.close()
Example #52
0
def hrc_gain_trend_plot():
    """
    create time trend of Gaussian profiles fit on HRC PHA data. It also create trend along radial distnace for each year
    Input:  none but the data is read from <house_keeping>/fitting_results
    Outut:  time trend plots in <plot_dir> / hrc_i_time_trend.png hrc_s_time_trend.png
            radial_distance plots            hrc_i_radial_dist_year<year>.png
    """

    file = house_keeping + 'fitting_results'
    f = open(file, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()
    #
    #--- set a few things such as labeling
    #
    yMinSets = [0, 0, 20]
    yMaxSets = [250, 250, 100]
    yname = 'PHA'
    entLabels = ["PHA Median", "PHA Voigt Peak Position", "PHA FWHM"]
    #
    #--- time trend plots
    #
    xmin = 1999
    ctemp = tcnv.currentTime()  #--- finding this year
    xmax = end_year = int(ctemp[0]) + 1
    xname = 'Time (Year)'
    #
    #--- HRC I
    #
    time_i = []
    dist_i = []
    med_i = []
    center_i = []
    width_i = []

    time_s = []
    dist_s = []
    med_s = []
    center_s = []
    width_s = []

    for ent in data:
        if ent[0] == '#':  #--- avoid comments
            continue

        atemp = re.split('\t+', ent)
        #
        #--- if the width is < 1 or amp is smaller than 5, probably it did not find a correct source position
        #
        if float(atemp[12]) < 1:
            continue
        if float(atemp[11]) < 5:
            continue

        fyear = tcnv.sectoFracYear(float(atemp[2]))
        if atemp[3] == 'HRC-I':
            time_i.append(fyear)
            dist_i.append(float(atemp[8]))
            med_i.append(float(atemp[9]))
            center_i.append(float(atemp[10]))
            width_i.append(float(atemp[12]))
        else:
            time_s.append(fyear)
            dist_s.append(float(atemp[8]))
            med_s.append(float(atemp[9]))
            center_s.append(float(atemp[10]))
            width_s.append(float(atemp[12]))

    xSets = [time_i, time_i, time_i]
    ySets = [med_i, center_i, width_i]
    dist = dist_indexing(dist_i)

    plotPanel(xmin, xmax, yMinSets, yMaxSets, xSets, ySets, dist, xname, yname,
              entLabels)
    cmd = 'mv out.png ' + web_dir + 'Trend_plots/hrc_i_time_trend.png'
    os.system(cmd)
    #
    #--- HRC S
    #
    xSets = [time_s, time_s, time_s]
    ySets = [med_s, center_s, width_s]
    dist = dist_indexing(dist_s)

    plotPanel(xmin, xmax, yMinSets, yMaxSets, xSets, ySets, dist, xname, yname,
              entLabels)
    cmd = 'mv out.png ' + web_dir + 'Trend_plots/hrc_s_time_trend.png'
    os.system(cmd)

    #
    #--- trend along radial distance
    #
    xmin = 0
    xmax = 16
    xname = 'Radial Distance(Arcsec)'
    #
    #--- HRC I
    #
    for pyear in range(1999, end_year):
        dist = []
        med = []
        center = []
        width = []
        mark = []
        pend = pyear + 1
        for j in range(0, len(time_i)):
            if (time_i[j] >= pyear) and (time_i[j] < pend):
                dist.append(dist_i[j])
                med.append(med_i[j])
                center.append(center_i[j])
                width.append(width_i[j])

        xSets = [dist, dist, dist]
        ySets = [med, center, width]
        plotPanel(xmin,
                  xmax,
                  yMinSets,
                  yMaxSets,
                  xSets,
                  ySets,
                  mark,
                  xname,
                  yname,
                  entLabels,
                  extraNote=pyear)

        outname = web_dir + '/Trend_plots/hrc_i_radial_dist_year' + str(
            pyear) + '.png'
        cmd = 'mv out.png ' + outname
        os.system(cmd)
#
#--- HRC S
#
    for pyear in range(1999, end_year):
        dist = []
        med = []
        center = []
        width = []
        mark = []
        pend = pyear + 1
        for j in range(0, len(time_s)):
            if (time_s[j] >= pyear) and (time_s[j] < pend):
                dist.append(dist_s[j])
                med.append(med_s[j])
                center.append(center_s[j])
                width.append(width_s[j])

        xSets = [dist, dist, dist]
        ySets = [med, center, width]
        plotPanel(xmin,
                  xmax,
                  yMinSets,
                  yMaxSets,
                  xSets,
                  ySets,
                  mark,
                  xname,
                  yname,
                  entLabels,
                  extraNote=pyear)

        outname = web_dir + '/Trend_plots/hrc_s_radial_dist_year' + str(
            pyear) + '.png'
        cmd = 'mv out.png ' + outname
        os.system(cmd)
def write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos, asig1, asig2, asig3, \
               mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3):

    """
    write a html page:
    input: ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos,  max_acc,max_apos,m10_acc
           m10_apos,mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos
    """

    (lyear, lmon, lday, lhours, lmin, lsec, lweekday, lyday, dst) = tcnv.currentTime('Local')

    f = open('acis.html', 'w')

#
#--- this is a html 5 document
#
    f.write('<!DOCTYPE html>\n')
    f.write('<html>\n')
    f.write('<head>\n')
    line = '<title>ACIS ' + ccd.upper() + ' Section ' + str(sec) + ' History Data</title>\n'
    f.write(line)
    f.write("<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />")
#
#--- css style sheet
#
    f.write('<style type="text/css">\n')
    f.write('table{text-align:center;margin-left:auto;margin-right:auto;border-style:solid;border-spacing:8px;border-width:2px;border-collapse:separate}\n')
    f.write('td {text-align:center}\n')
    f.write('a:link {color:aqua;}\n')
    f.write('a:visited {color:green;}\n')
    f.write('</style>\n')

    f.write('</head>\n')

    f.write('<body style="color:#FFFFFF;background-color:black"> \n')

    line = '<br /><h3> Last Update: ' + str(lmon) + '/' + str(lday) + ' / ' +  str(lyear) + '</h3>\n'
    f.write(line)
    f.write('<table border=1>\n')

    header_write(f)

    for i in range(0, len(year)):
        f.write('<tr>\n')
        line =        '<td>' + str(year[i])     + '</td>\t'
        line = line + '<td>' + str(month[i])    + '</td>\t'
        line = line + '<td>' + str(mean_dff[i]) + '</td>\t'
        line = line + '<td>' + str(std_dff[i])  + '</td>\t'
        line = line + '<td>' + str(min_dff[i])  + '</td>\t'
        line = line + '<td>' + str(min_dpos[i]) + '</td>\t'
        line = line + '<td>' + str(max_dff[i])  + '</td>\t'
        line = line + '<td>' + str(max_dpos[i]) + '</td>\t'
        line = line + '<td>' + str(dsig1[i])    + '</td>\t'
        line = line + '<td>' + str(dsig2[i])    + '</td>\t'
        line = line + '<td>' + str(dsig3[i])    + '</td>\t'
        f.write(line)
        syear = str(year[i])
        smon  = str(month[i])
        if month[i] < 10:
            smon = '0' + smon
        lccd = ccd.replace('_', '')
        file = 'ACIS_' + smon + '_' + syear + '_' + lccd 

        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/' + file + '.fits.gz">fits</a></td>\n'
        f.write(line)
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td>\n\n'
        f.write(line)

        line =        '<td>' + str(mean_acc[i]) + '</td>\t'
        line = line + '<td>' + str(std_acc[i])  + '</td>\t'
        line = line + '<td>' + str(min_acc[i])  + '</td>\t'
        line = line + '<td>' + str(min_apos[i]) + '</td>\t'
        line = line + '<td>' + str(max_acc[i])  + '</td>\t'
        line = line + '<td>' + str(max_apos[i]) + '</td>\t'
        line = line + '<td>' + str(asig1[i])    + '</td>\t'
        line = line + '<td>' + str(asig2[i])    + '</td>\t'
        line = line + '<td>' + str(asig3[i])    + '</td>\n'
        f.write(line)

        file = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd 
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/' + file + '.fits.gz">fits</a></td>\n'
        f.write(line)
        line = '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' + file + '.png">map</a></td></tr>\n\n'
        f.write(line)
#
#--- put header every new year so that we can read data easier
#
        if month[i] % 12 == 0 and i != (len(year)-1):
            header_write(f)
            f.write('\n')

    f.write('</table>\n\n')
    f.write('<br /><br /><hr /><br />\n')

    line = '<br /><strong style="font-size:105%;float:right">Last Update: ' + smon + '/' + str(lday) + '/' + str(lyear) + '</strong>\n'
    f.write(line)

    line = '<p>If you have any questions about this page, contact <a href="mailto:[email protected]">[email protected].</a></p>\n'
    f.write(line)
    f.write('</body>\n')
    f.write('</html>\n')

    f.close()
Example #54
0
def update_new_obs_list():

    """
    update new_obs_lsit, too_list, ddt_list, and obs_in_30days
    no input but data are taken from sot_ocat.out.

    """
#
#--- set limit to the last 30  days
#
    [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime('LOCAL')
    tdom = tcnv.findDOM(year, mon, day, 0, 0, 0)
    dom_limit = tdom - 30                      
    dom_limit = tdom - 100                      
    dom30days = tdom + 30                           #---- use to find observations will happen in 30 days
#
#--- read special obsid --- poc list
#
    line = too_dir + 'special_obsid_poc_list'
    f    = open(line, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    sp_obsid = []
    sp_user  = []
    for ent in data:
        atemp = re.split('\s+', ent)
        sp_obsid.append(atemp[0])
        sp_user.append(atemp[1])
#
#--- read database
#
    line = obs_ss + 'sot_ocat.out'
    f    = open(line, 'r')
    data = [line.strip() for line in f.readlines()]
    f.close()

    out_list = []
#
#--- read too_list and ddt_list
#
    line = too_dir + 'too_list'
    fo   = open(line, 'r')
    too  = [line.strip() for line in fo.readlines()]
    fo.close()
    too_dict = {}
    for ent in too:
        atemp = re.split('\s+', ent)
        too_dict[atemp[2]] = ent
    
    line = too_dir + 'ddt_list'
    fo   = open(line, 'r')
    ddt  = [line.strip() for line in fo.readlines()]
    fo.close()
    ddt_dict = {}
    for ent in ddt:
        atemp = re.split('\s+', ent)
        ddt_dict[atemp[2]] = ent
#
#--- open temporary writing files
#
    newf = temp_dir + 'new_obs_list'
    out1 = open(newf, 'w')
    obsf = temp_dir + 'obs_in_30days'
    out2 = open(obsf, 'w')
#
#--- start itelations
#
    for ent in data:
        atemp  = re.split('\^', ent)
        try:
            status = atemp[16].strip().lower()
            date   = atemp[13].strip()
        except:
            continue
#
#--- limit data re-checking to the last 30 days
#
        if str(date).lower() == 'null' or str(date).lower() == 'none':
            dom = 'NA'
        else:
            temp  = re.split('\s+', str(date))
            omon  = tcnv.changeMonthFormat(temp[0])
            oday  = int(temp[1])
            oyear = int(temp[2])
            dom   = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)

        ochk = 0
        if str(date).lower() == 'null' or str(date).lower() == 'none':
            ochk = 1

        else:
            if dom > dom_limit:
                ochk = 1

        if ochk == 1  and (status == 'scheduled' or status == 'unobserved' or status == 'observed'):
            chk   = 0
            obsid = atemp[1].strip()
#
#--- check this obsid is listed on a special_obsid_poc_list
#
            sp_poc = 'na'
            for sval in range(0, len(sp_obsid)):
                if obsid == sp_obsid[sval]:
                    sp_poc = sp_user[sval]
                    break
#
#--- extract basic information
#
            monitor = []
            groupid = []
            try:
                (group_id, pre_id, pre_min_lead, pre_max_lead, grating, type, instrument, obs_ao_str, status, \
                seq_nbr, ocat_propid, soe_st_sched_date, lts_lt_plan,targname, object) = sql.get_target_info(int(obsid), monitor,groupid)
    
                if soe_st_sched_date is not None:
                    date = soe_st_sched_date
                    chk = 1
                elif lts_lt_plan is not None:
                    date = lts_lt_plan
                    chk  = 1
                else:
                    date = 'NA'
                    chk = 0
            except:
                date = 'NA'
                chk = 0
#
#--- check status change
#
            if status == 'scheduled' or status == 'unobserved' or status == 'observed':
#
#--- recompute with updated date
#
                if date == 'NA':
                    dom = 'NA'
                else:
                    temp  = re.split('\s+', str(date))
                    omon  = tcnv.changeMonthFormat(temp[0])
                    oday  = int(temp[1])
                    oyear = int(temp[2])
                    dom   = tcnv.findDOM(oyear, omon, oday, 0, 0, 0)
#
#--- if it is ddt or  too, add to the list anyway
#
                if type.lower() == 'ddt' or type.lower() == 'too':
                    chk = 1
                    if date == 'NA':
                        dom = tdom + 1000
#
#--- the observation is cleared all criteria;  prepare to print them out to files
#
                if chk == 1:
                    pchk = 1
                else:
                    pchk = 0
                    continue
    
                if sp_poc != 'na':
#
#-- for the case the obsid is given a specific poc
#
                    person = sp_poc

                elif type.lower() in ('ddt', 'too'):
                    test = pre_assigned_pos(object, grating)
                    if test != 0:
                        person = test
                    else:
#
#-- too/ddt case: assign poc
#
                        if date == 'NA':
                            person = 'TBD'
                            try:
                                [person, chk]  = tdfnc.find_too_ddt_poc(obsid)
                            except:
                                person = 'TBD'
                        else:
                            try:
                                [person, chk]  = tdfnc.find_too_ddt_poc(obsid)
                            except:
                                person = 'TBD'
#
#--- observed obsid but no poc name -- record mistake; so drop
#
                            if person  == 'TBD':
                                if status == 'observed':
                                    pchk = 0
                else:
#
#--- none too/ddt observations
#
                    person = tdfnc.match_usint_person(type,grating,int(seq_nbr),instrument, targname)
#
#--- print out to files
#
                if pchk == 1:
                    line  = type.lower() + '\t' + str(seq_nbr) + '\t' + obsid + '\t' + status  + '\t'   \
                            + person + '\t' + str(obs_ao_str) + '\t' + str(date) + '\n'
#
#--- if it is ddt or too observation, replace the line in ddt_list or too_list
#
                    if type.lower() == 'ddt':
                        try:
                            line = ddt_dict[obsid]
                            line = line + '\n'
                        except:
                            pass

                    if type.lower() == 'too':
                        try:
                            line = too_dict[obsid]
                            line = line + '\n'
                        except:
                            pass

                    out1.write(line)
    
                    if status != 'observed' and dom < dom30days:
                        out2.write(line)

    out1.close()
    out2.close()

    completeTask(temp_dir, too_dir, 'new_obs_list')
    completeTask(temp_dir, too_dir, 'obs_in_30days')

#
#---- create new_obs_list.txt
#
    ofile = too_dir + 'new_obs_list.txt'
    out   = open(ofile, 'w')
    out.write('Type    Seq #   ObsId   Status          TOO     AO      Observation Date\n')
    out.write('--------------------------------------------------------------------------\n\n')
    out.close()
    cmd = 'cat ' + too_dir + 'new_obs_list >> ' + ofile
    os.system(cmd)
def acis_dose_make_data_html(indir = 'NA', outdir = 'NA', comp_test = 'NA'):

    """
    read data and create html pages
    input: indir, outdir, both are optional
    """
#
#--- setting indir and outdir if not given
#
    if indir   == 'NA':
        if comp_test == 'test':
            indir   = test_data_out
        else:
            indir   = data_out

    if outdir  == 'NA':
        if comp_test == 'test':
            outdir  = test_data_out
        else:
            outdir  = data_out

#
#--- read data
#
    for ccd in ('i_2', 'i_3', 's_2', 's_3'):
        for sec in range(0, 4):

            inst = ccd + '_n_' + str(sec)


            date     = []
            year     = []
            month    = []
            mean_acc = []
            std_acc  = []
            min_acc  = []
            min_apos = []
            max_acc  = []
            max_apos = []
            asig1    = []
            asig2    = []
            asig3    = []
            mean_dff = []
            std_dff  = []
            min_dff  = []
            min_dpos = []
            max_dff  = []
            max_dpos = []
            dsig1    = []
            dsig2    = []
            dsig3    = []
        
            [date,year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff, \
              min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = expf.readExpData(indir, inst)

#
#--- write html page
#
            write_html(ccd, sec, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, \
                                 mean_dff,std_dff,min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3)

            out_name = outdir + inst + '.html'
            cmd = 'mv acis.html ' + out_name
            os.system(cmd)
#
#--- update top html page
#
    line = house_keeping + 'exposure.html'
    f    = open(line, 'r')
    data = f.read()
    f.close()

    now   = tcnv.currentTime("Display")
    ndate = 'Last Update: ' + now
    ndata = data.replace("Last Update:", ndate)


    line  = web_dir + 'exposure.html'
    f     = open(line, 'w')
    f.write(ndata)
    f.close()
#
#--- update plot page htmls
#
    update_plt_html_date(comp_test)
Example #56
0
def create_monthly(year='', mon=''):
    """
    create monthly report
    input:  year    --- year; if it is "", the year of the last month will be used
            mon     --- month;if it is "", the month value of the last month is used
    output: monthly report in /data/mta4/www/REPORTS/MONTLHY/<yyyy><MMM>/
    """
    #
    #--- if year and month are not given, use the last month's month and year value
    #
    if year == '':
        #
        #--- find today's date
        #
        ltime = tcnv.currentTime()
        year = ltime[0]
        lyear = str(year)  #--- '2016'
        #
        #--- set the last month's month and year
        #
        mon = ltime[1] - 1

        if mon < 1:
            mon = 12
            year -= 1
            lyear = str(year)  #--- '2016'

    cmon = str(mon)
    if mon < 10:
        cmon = '0' + cmon  #--- e.g. 03 or 11

    lmon = tcnv.changeMonthFormat(mon)  #--- e.g. Mar or Nov

    lmonyr = lmon.lower() + lyear[2] + lyear[3]  #--- e.g. jan16
    lm_y = cmon + '_' + lyear  #--- e.g. 03_2016
    #
    #--- set output directory
    #
    odir = '/data/mta4/www/REPORTS/MONTHLY/' + str(year) + lmon.upper()

    cmd = 'mkdir ' + odir
    os.system(cmd)

    odir = odir + '/'
    #
    #--- set month interval depending on leap year or not
    #
    if tcnv.isLeapYear(year) == 0:
        sdate = [
            '001', '032', '060', '091', '121', '152', '182', '213', '244',
            '274', '305', '335'
        ]
        edate = [
            '032', '060', '091', '121', '152', '182', '213', '244', '274',
            '305', '335', '001'
        ]
    else:
        sdate = [
            '001', '032', '061', '092', '122', '153', '183', '214', '245',
            '275', '306', '336'
        ]
        edate = [
            '032', '061', '092', '122', '153', '183', '214', '245', '275',
            '306', '336', '001'
        ]
#
#--- set start and stop time of the month in <yyyy>:<ddd>:00:00:00 format
#
    tstart = str(year) + ':' + sdate[mon - 1] + ':00:00:00'

    eyear = year
    if mon == 12:
        eyear = year + 1
    tstop = str(eyear) + ':' + edate[mon - 1] + ':00:00:00'
    #
    #--- create configulation plot
    #
    cmd = "cd /data/mta/Script/Month/Config/; create_config_plot.py " + tstart + ' ' + tstop
    os.system(cmd)
    #
    #--- create CTI plots
    #
    os.system(
        "cd /data/mta/Script/Month/CTI/;    monthly_report_cti_avg_plots.py")
    os.system(
        "cd /data/mta/Script/Month/CTI/;    monthly_report_cti_avg_plots_two_section.py"
    )
    #
    #--- create Focal Plane temperature plots
    #
    os.system("cd /data/mta/Script/Month/FOCAL/;  run_all_focal_scripts")
    #
    #--- create ACIS SIB plots
    #
    os.system("cd /data/mta/Script/Month/SIB/;    sib_monthly_report_plot.py")
    #
    #--- create SIM movement plots
    #
    os.system("cd /data/mta/Script/Month/SIM/;    create_monthly_sim_plots")
    #
    #--- copy the created plots to the report directory
    #
    cmd = "cp /data/mta/Script/Month/Config/rad_use_*.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/Script/Month/CTI/Plots/*png " + odir
    os.system(cmd)
    cmd = "cp -rf /data/mta/Script/Month/CTI/Data " + odir
    os.system(cmd)
    cmd = "cp /data/mta/Script/Month/FOCAL/*png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/Script/Month/FOCAL/Plots/*png " + odir
    os.system(cmd)
    cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_ccd_plot_front_side.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta_www/mta_bad_pixel/Plots/hist_plot_ccd5.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/Script/Month/SIB/*png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
    os.system(cmd)
    cmd = "cp /data/mta_www/mta_grat/Focus/Plots/acis_hetg_streak_lrf_focus.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta_www/mta_grat/Focus/Plots/acis_hetg_ax_lrf_focus.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/Script/Month/SIM/*.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta4/www/DAILY/mta_rad/mon_per_diff_last_one_year.gif " + odir
    os.system(cmd)
    cmd = "cp /data/mta4/www/DAILY/mta_rad/rad_cnts_" + lmonyr + ".gif " + odir
    os.system(cmd)
    cmd = "cp /data/mta4/www/DAILY/mta_pcad/IRU/Plots_new/" + lyear + '/' + lmonyr + "_bias.png " + odir
    os.system(cmd)
    cmd = "cp /data/mta/www/mta_max_exp/Images/hrc_max_exp.gif " + odir
    os.system(cmd)
    #
    #--- move the plots to past plot saving directories
    #
    os.system(
        "mv -f /data/mta/Script/Month/Config/*.png /data/mta/Script/Month/Config/Plots/."
    )
    os.system(
        "mv -f /data/mta/Script/Month/FOCAL/Plots/*gif /data/mta/Script/Month/FOCAL/Plots/Past/."
    )
    os.system(
        "mv -f /data/mta/Script/Month/SIB/*png /data/mta/Script/Month/SIB/Plots/."
    )
    #
    #--- sun spot cycle download
    #
    cmd = 'wget -q -O' + odir + '/solar-cycle-sunspot-number.gif  http://services.swpc.noaa.gov/images/solar-cycle-sunspot-number.gif'
    os.system(cmd)
    #
    #--- now copy  month depend plot files and set the template file for the month
    #
    if mon == 1 or mon == 7:
        cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/I3cp.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_sci_run/Corner_pix/Trend_Plots/S3cp.gif " + odir
        os.system(cmd)

        cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Overclock/ccd2.png " + odir + 'ccd2_oc.png'
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_bias_bkg/Plots/Sub/ccd2.png       " + odir + 'ccd2_sub.png'
        os.system(cmd)

        cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_2.png  " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_6.png  " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_7.png  " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_pcad/ACA/TOTAL/Report/MAG_I_AVG_11.png " + odir
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY1.html'  #--- template file name

    elif mon == 2 or mon == 8:
        cmd = "cp /data/mta4/www/DAILY/mta_src/ACIS-I_d_tyear10_psf.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_xy.gif           " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_rnd.gif        " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rnd.gif        " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_ravg.gif       " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_ravg.gif       " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_d_snr.gif        " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_snr.gif        " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_rot.gif          " + odir
        os.system(cmd)
        cmd = "cp /data/mta4/www/DAILY/mta_src/tot_all_t_rot.gif        " + odir
        os.system(cmd)

        cmd = "cp /data/mta_www/mta_aiming/Fig_save/acis_point_err.gif  " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_i_point_err.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_aiming/Fig_save/hrc_s_point_err.gif " + odir
        os.system(cmd)

        cmd = "cp -r /data/mta4/www/DAILY/mta_src/Plots " + odir + 'HRMA_Plots'
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY2.html'

    elif mon == 3 or mon == 9:
        cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_cti.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD3_rej_obs.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_cti.gif " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_sci_run/Events_rej/CCD7_rej_obs.gif " + odir
        os.system(cmd)

        cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd3.png   " + odir
        os.system(cmd)
        cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd3.png " + odir
        os.system(cmd)
        cmd = "cp /data/mta/www/mta_acis_gain/Plots/gain_plot_ccd5.png   " + odir
        os.system(cmd)
        cmd = "cp /data/mta/www/mta_acis_gain/Plots/offset_plot_ccd5.png " + odir
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY3.html'

    elif mon == 4 or mon == 10:
        #cmd = "cp /data/mta_www/mta_grat/EdE/heg_all.gif  " + odir
        #os.system(cmd)
        #cmd = "cp /data/mta_www/mta_grat/EdE/meg_all.gif  " + odir
        #os.system(cmd)
        #cmd = "cp /data/mta_www/mta_grat/EdE/leg_all.gif  " + odir
        #os.system(cmd)
        cmd = "cp /data/mta_www/mta_grat/EdE/*_2019.png " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sim_twist/Plots/twist_plot.png " + odir
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY4.html'

    elif mon == 5 or mon == 11:
        cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd3.png " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd5.png " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sib/Plots/Plot_long_term/full_plot_ccd7.png " + odir
        os.system(cmd)

        cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_pos.html   " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_width.html " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_acis_hist/Html_pages/acis_hist_cccd3_high_cnt.html   " + odir
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY5.html'

    elif mon == 6 or mon == 12:
        cmd = "cp /data/mta_www/mta_sim_twist/Plots/I-1.png   " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sim_twist/Plots/S-2.png   " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-I-2.png " + odir
        os.system(cmd)
        cmd = "cp /data/mta_www/mta_sim_twist/Plots/H-S-2.png " + odir
        os.system(cmd)

        file = '/data/mta/Script/Month/Scripts/Templates/MONTHLY6.html'
#
#--- create clean acis and hrc exposure maps using ds9
#
    run_exposure_maps(lyear, cmon)

    #--------------------------------------------------
    #--- read the template and substitute the contents
    #--------------------------------------------------

    fx = open(file, 'r')
    text = fx.read()
    fx.close()
    #
    #--- substitute values
    #
    fmonth_list = [
        'January', 'February', 'March', 'April', 'May', 'June', 'July',
        'August', 'September', 'October', 'November', 'December'
    ]

    fmon = fmonth_list[mon - 1]
    text = text.replace('#Month#', fmon)  #--- full month name e.g. May
    text = text.replace('#YEAR#', lyear)  #--- full year nane  e.g. 2016

    text = text.replace('#SMON#', lmon)  #--- short month, e.g. Jan
    text = text.replace('#LSMON#',
                        lmon.lower())  #--- short lower month, e.g. jan
    text = text.replace('#USMON#',
                        lmon.upper())  #--- short upper month, e.g. JAN

    line = 'rad_use_' + lmonyr + '.png'
    text = text.replace('#RADUSE#', line)  #--- e.g., rad_use_oct16.png

    line = cmon + '_' + lyear
    text = text.replace('#LMONYR#', line)  #--- mon_year e.g. 07_2016
    #
    #--- acis exposure tables
    #
    os.system(
        "cd /data/mta/Script/Exposure/Exc/; /data/mta/Script/Exposure/Scripts/ACIS_Scripts/acis_dose_monthly_report.py"
    )

    line = '/data/mta/Script/Exposure/Exc/monthly_diff_' + cmon + '_' + lyear
    f = open(line, 'r')
    dout = f.read()
    f.close()
    text = text.replace('#ACISMON#', dout)  #--- acis monthly dose

    line = '/data/mta/Script/Exposure/Exc/monthly_acc_' + cmon + '_' + lyear
    f = open(line, 'r')
    dout = f.read()
    f.close()
    text = text.replace('#ACISCMON#', dout)  #--- acis cumulative dose
    #
    #--- last 12 months exposure maps
    #
    byear = year - 1
    text = past_data_entry(1, 0, mon, byear, text)
    text = past_data_entry(2, 3, mon, byear, text)
    text = past_data_entry(3, 6, mon, byear, text)
    text = past_data_entry(4, 9, mon, byear, text)
    #
    #---- acis focal
    #
    fx = open('/data/mta/Script/Month/FOCAL/Plots/month_avg', 'r')
    data = [line.strip() for line in fx.readlines()]
    fx.close()

    atemp = re.split(':', data[0])
    btemp = re.split('\+\/\-', atemp[1])
    ft = round(float(btemp[0]), 2)
    ferr = round(float(btemp[1]), 2)

    text = text.replace('#FT#', str(ft))
    text = text.replace('#FERR#', str(ferr))

    atemp = re.split(':', data[1])
    btemp = re.split('\+\/\-', atemp[1])
    fw = round(float(btemp[0]), 2)
    werr = round(float(btemp[1]), 2)

    text = text.replace('#FW#', str(fw))
    text = text.replace('#WERR#', str(werr))

    line = 'erad_' + lmonyr + '.gif'  #--- erand_nov16.gif
    text = text.replace('#ERAND#', line)
    #
    #--- acis sib
    #
    yearmon = str(year) + '_' + cmon
    text = text.replace('#YEARMON#', yearmon)  #--- e.g. 2015_03
    #
    #--- hrc i monthly dose
    #
    line = hrc_monthly_report('HRCI', cmon, lmon, lyear)
    text = text.replace('#HRCIDOSE#', line)  #--- HRC I DIFF
    #
    #--- hrc cumulative
    #
    line = hrc_cumulative_report('HRCI', cmon, lmon, lyear)
    text = text.replace('#CHRCIDOSE#', line)  #--- HRC I Cumulative
    #
    #--- hrc s monthly dose
    #
    line = hrc_monthly_report('HRCS', cmon, lmon, lyear)
    text = text.replace('#HRCSDOSE#', line)  #--- HRC S DIFF
    #
    #--- hrc cumulative
    #
    line = hrc_cumulative_report('HRCS', cmon, lmon, lyear)
    text = text.replace('#CHRCSDOSE#', line)  #--- HRC S Cumulative
    #
    #--- IRU
    #
    text = text.replace("#SMONYR#", lmonyr)  #--- e.g., nov16
    #
    #--- envelope trending
    #
    line = get_envelope_trending(mon, odir)
    text = text.replace("#ENVTREND#", line)
    #
    #--- critical trends which are reported on Mar, Jun, Sep, and Dec
    #
    if mon in [3, 6, 9, 12]:
        text = run_critical_trend(text)
#
#--- Monthly Trend Reports
#
    text = run_month_trend(mon, text)
    #
    #--- last index
    #
    line = create_index(year, mon)
    text = text.replace('#YINDEX#', line)
    #
    #--- finally print out the report
    #
    ofile = odir + "MONTHLY.html"
    fo = open(ofile, 'w')
    fo.write(text)
    fo.close()
    #
    #--- chnage permission and owners
    #
    cmd = 'chmod 755 ' + odir + '/*'
    os.system(cmd)

    cmd = 'chgrp -R mtagroup ' + odir + '/*'
    os.system(cmd)
Example #57
0
def report_error():
    """
    read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email
    Input:  none but read from <cup_usr_list>_error_list
    Output: email sent out
    """

    #
    #--- find the current time
    #
    [year, mon, day, hours, min, sec, weekday, yday,
     dst] = tcnv.currentTime("Local")
    #
    #--- create surfix for files which will be saved in Past_errors directory
    #
    smon = str(mon)
    if mon < 10:
        smon = '0' + smon
    sday = str(day)
    if day < 10:
        sday = '0' + sday
    tail = str(year) + smon + sday

    for tag in cpu_usr_list:

        efile = house_keeping + 'Records/' + tag + '_error_list'
        pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail
        prev_line = ''

        chk = mcf.chkFile(efile)
        if chk > 0:
            #
            #--- read error messages from the file
            #
            f = open(efile, 'r')
            data = [line.strip() for line in f.readlines()]
            f.close()
            #
            #--- sort the data so that we can correct messages to each cron job together
            #
            data.sort()

            task_list = []
            time_list = []
            mssg_list = []
            for ent in data:
                atemp = re.split(' : ', ent)
                task_list.append(atemp[0])

                stime = int(atemp[1])
                dtime = tcnv.axTimeMTA(stime)
                time_list.append(dtime)

                mssg_list.append(atemp[2])
#
#--- write out cron job name
#
            fo = open(zspace, 'w')
            cname = task_list[0]
            line = '\n\n' + cname + '\n____________________\n\n'
            fo.write(line)

            for i in range(0, len(mssg_list)):
                if task_list[i] != cname:
                    cname = task_list[i]
                    line = '\n\n' + cname + '\n____________________\n\n'
                    fo.write(line)
#
#--- create each line. if it is exactly same as one line before, skip it
#
                line = time_list[i] + ' : ' + mssg_list[i] + '\n'

                if line != prev_line:
                    fo.write(line)
                prev_line = line

            fo.close()
            #
            #--- send email out
            #
            #            cmd = 'cp  ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag
            #            os.system(cmd)
            send_mail(tag, email_list)
            #
            #--- move the error list to Past_errors directory
            #
            cmd = 'mv ' + efile + ' ' + pfile
            os.system(cmd)
Example #58
0
def clean_table():
    """
    sort and clean the table data in <data_dir>/Results
    Input:  none but read from <data_dir>/Results/<elm>_ccd<ccd#>
    Output: cleaned up <data_dir>/Results/<elm>_ccd<ccd#>
    """
    #
    #--- make a backup
    #
    atime = tcnv.currentTime(format='UTC')
    tyear = str(atime[0])
    syear = tyear[2] + tyear[3]

    tmon = atime[1]
    smon = str(tmon)
    if int(tmon) < 10:
        smon = '0' + smon

    tday = atime[2]
    sday = str(tday)
    if int(tday) < 10:
        sday = '0' + sday

    cout = data_dir + '/Results/Save_' + smon + sday + syear
    mcf.mk_empty_dir(cout)
    cmd = 'cp -f  ' + data_dir + '/Results/*_ccd* ' + cout + '/.'
    os.system(cmd)
    #
    #--- now clean up the data
    #
    cmd = 'ls ' + data_dir + '/Results/*_ccd* > ' + zspace
    os.system(cmd)

    f = open(zspace, 'r')
    flist = [line.strip() for line in f.readlines()]
    f.close()
    mcf.rm_file(zspace)

    for file in flist:
        f = open(file, 'r')
        data = [line.strip() for line in f.readlines()]
        f.close()

        data.sort()

        prev = data[0]
        cleaned = [prev]

        for ent in data:
            if ent == prev:
                prev = ent
            else:
                cleaned.append(ent)
                prev = ent

        fo = open(file, 'w')
        for ent in cleaned:
            line = ent + '\n'
            fo.write(line)

        fo.close()