def get_data(msid, start, stop): """ extract data for the given data period input: msid --- msid start --- starting time in seconds from 1998.1.1 stop --- stopping time in seconds from 1998.1.1 output: data --- a list of msid values """ # #--- extract data with dataseeker # try: ecf.data_seeker(start, stop, msid) [col, tbdata] = ecf.read_fits_file('temp_out.fits') mcf.rm_file('temp_out.fits') except: return [] time = tbdata.field('time') # #--- if the dataseeker's data is not filled for the given data period #--- stop any farther data proccess # if time[-1] < 0.95 * stop: data = [] else: try: name = msid + '_avg' data = tbdata.field(name) except: data = [] return data
def find_pitch_angle(start, stop): """ create a table of time and sun pitch angle input: start --- starting time in seconds from 1998.1.1 stop --- stopping time in seconds from 1998.1.1 output: time --- a list of time in seconds from 1998.1.1 pitch --- a list of ptich in degree """ # #--- extract pcad aspsol fits files for the given time period # line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = pcad\n' line = line + 'subdetector = aca\n' line = line + 'level = 1\n' line = line + 'filetype =aspsol\n' line = line + 'tstart = ' + str(start) + '\n' line = line + 'tstop = ' + str(stop) + '\n' line = line + 'go\n' data_list = mcf.run_arc5gl_process(line) [cols, tbdata] = ecf.read_fits_file(data_list[0]) for ent in data_list: cmd = 'rm -rf ' + ent os.system(cmd) time = tbdata['time'] ra = tbdata['ra'] dec = tbdata['dec'] time_list = [] pitch_list = [] prev = 0.0 m = 0 for k in range(0, len(time)): # #--- select one data every 5 mins # itime = int(time[k]) if itime == prev: continue if m % 300 == 0: # #--- get the sun angle # pitch = find_chandra_pitch(time[k], ra[k], dec[k]) time_list.append(itime) pitch_list.append(pitch) prev = itime m += 1 time = numpy.array(time_list) pitch = numpy.array(pitch_list) return [time, pitch]
def get_data(start, stop, year, msid_list): print str(start) + '<-->' + str(stop) line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ephin\n' line = line + 'level = 0\n' line = line + 'filetype =ephhk \n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) data_list = ecf.read_file_data('ztemp_out') data_list = data_list[1:] # #--- uppend the data to the local fits data files # for fits in data_list: [cols, tbdata] = ecf.read_fits_file(fits) time = tbdata['time'] for col in msid_list: # #--- ignore columns with "ST_" (standard dev) and time # mdata = tbdata[col] cdata = [time, mdata] ocols = ['time', col.lower()] if not os.path.isdir(out_dir): cmd = 'mkdir ' + out_dir os.system(cmd) ofits = out_dir + col.lower() + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata) mcf.rm_file(fits)
def get_data(start, stop, year, msid_list, out_dir): """ update eph l1 related data for the given data peirod input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 year --- data extracted year msid_list --- list of msids out_dir --- output_directory output: <out_dir>/<msid>_full_data_<year>.fits """ print(str(start) + '<-->' + str(stop)) line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ephin\n' line = line + 'level = 0\n' line = line + 'filetype =ephhk \n' line = line + 'tstart = ' + str(start) + '\n' line = line + 'tstop = ' + str(stop) + '\n' line = line + 'go\n' data_list = mcf.run_arc5gl_process(line) # #--- uppend the data to the local fits data files # for fits in data_list: [cols, tbdata] = ecf.read_fits_file(fits) time = tbdata['time'] for col in msid_list: # #--- ignore columns with "ST_" (standard dev) and time # mdata = tbdata[col] cdata = [time, mdata] ocols = ['time', col.lower()] if not os.path.isdir(out_dir): cmd = 'mkdir ' + out_dir os.system(cmd) ofits = out_dir + col.lower() + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata) mcf.rm_files(fits)
def get_data(start, stop, year, msid_list): print(str(start) + '<-->' + str(stop)) line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ephin\n' line = line + 'level = 0\n' line = line + 'filetype =ephhk \n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' data_list = mcf.run_arc5gl_process(line) # #--- uppend the data to the local fits data files # for fits in data_list: [cols, tbdata] = ecf.read_fits_file(fits) time = tbdata['time'] for col in msid_list: # #--- ignore columns with "ST_" (standard dev) and time # mdata = tbdata[col] cdata = [time, mdata] ocols = ['time', col.lower()] if not os.path.isdir(out_dir): cmd = 'mkdir ' + out_dir os.system(cmd) ofits = out_dir + col.lower() + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata) mcf.rm_files(fits)
def update_eph_data(date=''): """ collect grad and comp data for trending input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # #sfile = 'eph_list' #glist = ecf.read_file_data(sfile) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() day_list = [] for year in range(1999, 2018): #---- CHANGE CHANGE CHAGE!!!!! lyear = year for mon in range(1, 13): if year == 2016 and mon < 9: continue if year == 2017 and mon > 10: continue cmon = str(mon) if mon < 10: cmon = '0' + cmon nmon = mon + 1 if nmon > 12: nmon = 1 lyear += 1 clmon = str(nmon) if nmon < 10: clmon = '0' + clmon start = str(year) + '-' + cmon + '-01T00:00:00' stop = str(lyear) + '-' + clmon + '-01T00:00:00' print "Period: " + str(start) + "<--->" + str(stop) for group in glist: print "Group: " + group # #---CHANGE THE DETECTOR/FILETYPE BEFORE RUNNING IF IT IS DIFFERENT FROM EPHHK # line = 'operation=retrieve\n' line = line + 'dataset=flight\n' line = line + 'detector=ephin\n' line = line + 'level=0\n' line = line + 'filetype=epheio\n' line = line + 'tstart=' + start + '\n' line = line + 'tstop=' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(1, len(cols)): # #--- select col name without ST_ (which is standard dev) # col = cols[k] mc = re.search('ST_', col) if mc is not None: continue mc = re.search('quality', col, re.IGNORECASE) if mc is not None: continue mc = re.search('mjf', col, re.IGNORECASE) if mc is not None: continue mc = re.search('gap', col, re.IGNORECASE) if mc is not None: continue mc = re.search('dataqual', col, re.IGNORECASE) if mc is not None: continue mc = re.search('tlm_fmt', col, re.IGNORECASE) if mc is not None: continue # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, data, glim)
def update_compdatabase(): """ update comp related data using mp data input: none but read from /data/mta_www/mp_reports and <house_keeping>/<comp_group>_past output: <out_dir>/<comp msid>_full_data.fits """ for comp_group in comp_entry: # #--- read the last set of the input data and find the last entry # past = house_keeping + comp_group + '_past' past = mcf.read_data_file(past) last = past[-1] # #--- find today's data entry # cmd = 'ls /data/mta_www/mp_reports/*/' + comp_group + '/data/mta*fits* >' + zspace os.system(cmd) current = mcf.read_data_file(zspace) cmd = 'mv ' + zspace + ' ' + house_keeping + comp_group + '_past' os.system(cmd) # #--- find the data which are not read # new_fits = [] chk = 0 for ent in current: if chk == 0: if ent == last: chk = 1 continue new_fits.append(ent) # #--- uppend the data to the local fits data files # for fits in new_fits: [cols, tbdata] = ecf.read_fits_file(fits) time = tbdata['time'] for col in cols: # #--- ignore columns with "ST_" (standard dev) and time # if col.lower() == 'time': continue mc = re.search('st_', col.lower()) if mc is not None: continue mdata = tbdata[col] cdata = [time, mdata] ocols = ['time', col.lower()] ofits = out_dir + col.lower() + '_full_data.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata)
def extract_hrcveto_data(): """ extract hrc veto data input: none output: fits file data related to grad and comp """ # #--- read group names which need special treatment # glist = ['Hrcveto'] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() # #--- find the date to be filled # day_list = find_the_last_entry_time() for sday in day_list: print "Date: " + sday start = sday + 'T00:00:00' stop = sday + 'T23:59:59' for group in glist: print "Group: " + group line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = hrc\n' line = line + 'level = 0\n' line = line + 'filetype = hrcss\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cols = ['TLEVART', 'VLEVART', 'SHEVART'] cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for col in cols: # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, data, glim)
def get_data(tstart, tstop, year, grad_list, out_dir): """ update msid data in msid_list for the given data period input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 year --- the year in which data is extracted grad_list --- a list of group name in grads out_dir --- output_directory """ print("Period: " + str(tstart) + '<-->' + str(tstop) + ' in Year: ' + str(year)) # #--- extract ecach group data # for group in grad_list: print(group) line = 'operation=retrieve\n' line = line + 'dataset = mta\n' line = line + 'detector = grad\n' line = line + 'level = 0.5\n' line = line + 'filetype = ' + group + '\n' line = line + 'tstart = ' + str(tstart) + '\n' line = line + 'tstop = ' + str(tstop) + '\n' line = line + 'go\n' data_list = mcf.run_arc5gl_process(line) # #--- read the first fits file and prep for the data list # [cols, tbdata] = ecf.read_fits_file(data_list[0]) col_list = [] for ent in cols: if ent.lower() == 'time': continue mc = re.search('st_', ent.lower()) if mc is not None: continue col_list.append(ent) mcf.rm_files(data_list[0]) tdata = tbdata['time'] mdata = [] for col in col_list: mdata.append(tbdata[col]) # #--- read the rest of the data # clen = len(col_list) for k in range(1, len(data_list)): fits = data_list[k] [cols, tbdata] = ecf.read_fits_file(fits) tdata = numpy.append(tdata, tbdata['time']) for m in range(0, clen): cdata = tbdata[col_list[m]] mdata[m] = numpy.append(mdata[m], cdata) mcf.rm_files(fits) dout = out_dir + group.capitalize() + '/' if not os.path.isdir(dout): cmd = 'mkdir ' + dout os.system(cmd) # #--- write out the data to fits file # for k in range(0, clen): col = col_list[k] ocols = ['time', col.lower()] cdata = [tdata, mdata[k]] ofits = dout + col.lower() + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata) # #--- zip the fits file from the last year at the beginning of the year # ecf.check_zip_possible(dout)
def update_grad_and_comp_data(date=''): """ collect grad and comp data for trending input: date ---- the data colletion end date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # sfile = house_keeping + 'mp_process_list' glist = ecf.read_file_data(sfile) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- find date to read the data # if date == '': yesterday = datetime.date.today() - datetime.timedelta(1) yesterday = str(yesterday).replace('-', '') date_list = find_the_last_entry_time(yesterday) else: date_list = [date] for day in date_list: # #--- find the names of the fits files of the day of the group # print "Date: " + str(day) for group in glist: print "Group: " + str(group) cmd = 'ls /data/mta_www/mp_reports/' + day + '/' + group + '/data/mta*fits* > ' + zspace os.system(cmd) flist = ecf.read_file_data(zspace, remove=1) # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- read out the data for the full day # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(1, len(cols)): # #--- select col name without ST_ (which is standard dev) # col = cols[k] mc = re.search('ST_', col) if mc is not None: continue # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = ecf.convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, data, glim)
def update_mta_comp_database(): """ updata database of mta computed msids input: none but read from /data/mta4/Deriv/*fits files outpu: updated data file: <data_dir>/<msid>_data """ # #--- get a list of data fits file names # infile = house_keeping + 'mta_comp_fits_files' data = ecf.read_file_data(infile) for fits in data: # #--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is #--- if this is not hrc (or hrc all), tail = 2 # mc = re.search('hrc', fits) if mc is not None: atemp = re.split('_', fits) btemp = re.split('.fits', atemp[1]) tail = btemp[0] else: tail = 2 [cols, tbdata] = ecf.read_fits_file(fits) time = [] for ent in tbdata.field('time'): stime = float(ent) # #--- check whether the time is in dom # if stime < 31536000: stime = ecf.dom_to_stime(float(ent)) time.append(stime) for col in cols: col = col.lower() # #--- we need only *_avg columns # mc = re.search('_avg', col) if mc is not None: vals = tbdata.field(col) ctime = [] cvals = [] for m in range(0, len(time)): # #--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999) # if str(vals[m]) in ['nan', 'NaN', 'NAN']: continue nval = float(vals[m]) if nval in [-999, -998, -99, 99, 998, 999]: continue else: ctime.append(time[m]) cvals.append(nval) atemp = re.split('_', col) msid = atemp[-2] if mcf.chkNumeric(tail): oname = msid else: oname = msid + '_' + tail print "MSID: " + str(oname) cmd = 'rm ' + data_dir + oname + '_data' os.system(cmd) # #--- read limit table for the msid # l_list = ecf.set_limit_list(msid) if len(l_list) == 0: try: l_list = mta_db[msid] except: l_list = [] update_data(msid, l_list, dset = tail, time=ctime, vals=cvals)
def update_simsuppl_data(date=''): """ collect sim diag msids input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to simdiag """ # #--- read group names which need special treatment # sfile = house_keeping + 'msid_list_simactu_supple' data = ecf.read_file_data(sfile) cols = [] g_dir = {} for ent in data: atemp = re.split('\s+', ent) cols.append(atemp[0]) g_dir[atemp[0]] = atemp[1] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() # #--- find date to read the data # if date == '': yesterday = datetime.date.today() - datetime.timedelta(1) yesterday = str(yesterday).replace('-', '') date_list = find_the_last_entry_time(yesterday) else: date_list = [date] for sday in date_list: print "Date: " + sday start = sday + 'T00:00:00' stop = sday + 'T23:59:59' line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = sim\n' line = line + 'level = 0\n' line = line + 'filetype = sim\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols_xxx, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(0, len(cols)): # #--- select col name without ST_ (which is standard dev) # col = cols[k] # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, g_dir[msid], dtime, data, glim)
def recover_hrcveto_data(): """ recover hrc veto data input: none output: fits file data related to grad and comp """ # #--- read group names which need special treatment # #sfile = 'eph_list' #glist = ecf.read_file_data(sfile) glist = ['Hrcveto'] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() day_list = [] for year in range(1999, 2018): lyear = year cyear = str(year) for mon in range(1, 13): if year == 1999: if mon < 8: continue if year == 2017: if mon > 10: break cmon = str(mon) if mon < 10: cmon = '0' + cmon nmon = mon + 1 if nmon > 12: nmon = 1 lyear += 1 cnmon = str(nmon) if nmon < 10: cnmon = '0' + cnmon start = str(year) + '-' + cmon + '-01T00:00:00' stop = str(lyear) + '-' + cnmon + '-01T00:00:00' for group in glist: print "Group: " + group + ' : ' + str(start) + '<-->' + str( stop) line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = hrc\n' line = line + 'level = 0\n' line = line + 'filetype = hrcss\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cols = ['TLEVART', 'VLEVART', 'SHEVART'] cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for col in cols: # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, data, glim)
def dea_full_data_update(chk): """ update deahk search database input: chk --- whether to request full data update: chk == 1:yes output: <deposit_dir>/Deahk/<group>/<msid>_full_data_<year>fits """ tyear = int(float(time.strftime("%Y", time.gmtime()))) cmd = 'ls ' + data_dir + 'Deahk_*/*_week_data.fits > ' + zspace os.system(cmd) data = mcf.read_data_file(zspace, remove=1) for ent in data: atemp = re.split('\/', ent) group = atemp[-2] btemp = re.split('_', atemp[-1]) msid = btemp[0] print("MSID: " + str(msid) + ' in ' + group) [cols, tbdata] = ecf.read_fits_file(ent) time = tbdata['time'] tdata = tbdata[msid] cols = ['time', msid] # #--- regular data update # if chk == 0: # #--- normal daily data update # ofits = deposit_dir + 'Deahk_save/' + group + '/' + msid + '_full_data_' ofits = ofits + str(tyear) + '.fits' if os.pathisfile(ofits): ltime = ecf.find_the_last_entry_time(ofits) ctime = str(tyear + 1) + ':001:00:00:00' nchk = 0 # #--- if the data is over the year boundray, fill up the last year and create a new one for the new year # else: ofits = deposit_dir + 'Deahk_save/' + group + '/' + msid ofits = ofits + '_full_data_' + str(tyear - 1) + '.fits' nfits = deposit_dir + 'Deahk_save/' + group + '/' + msid nfits = nfits + '_full_data_' + str(tyear) + '.fits' ltime = ecf.find_the_last_entry_time(ofits) ctime = str(tyear) + ':001:00:00:00' nchk = 1 select = [(time > ltime) & (time < ctime)] stime = time[select] sdata = tdata[select] cdata = [stime, sdata] ecf.update_fits_file(ofits, cols, cdata) if nchk > 0: select = [time >= ctime] stime = time[select] sdata = tdata[select] cdata = [stime, sdata] ecf.create_fits_file(nfits, cols, cdata) # #--- start from beginning (year 1999) # else: for year in range(1999, tyear + 1): tstart = str(year) + ':001:00:00:00' tstart = Chandra.Time.DateTime(tstart).secs tstop = str(year + 1) + ':001:00:00:00' tstop = Chandra.Time.DateTime(tstop).secs select = [(time >= tstart) & (time < tstop)] stime = time[select] sdata = tdata[select] cdata = [stime, sdata] out = deposit_dir + 'Deahk_save/' + group + '/' if not os.path.isdir(out): cmd = 'mkdir ' + out out = out + msid + '_full_data_' + str(year) + '.fits' ecf.create_fits_file(out, cols, cdata)
def get_data(start, stop, year, msid_list, out_dir): """ update eph l1 related data for the given data peirod input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 year --- data extracted year msid_list --- list of msids out_dir --- output_directory output: <out_dir>/<msid>_full_data_<year>.fits """ print str(start) + '<-->' + str(stop) line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ephin\n' line = line + 'level = 0\n' line = line + 'filetype =ephhk \n' line = line + 'tstart = ' + str(start) + '\n' line = line + 'tstop = ' + str(stop) + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) data_list = ecf.read_file_data('ztemp_out', remove=1) data_list = data_list[1:] # #--- uppend the data to the local fits data files # for fits in data_list: [cols, tbdata] = ecf.read_fits_file(fits) time = tbdata['time'] for col in msid_list: # #--- ignore columns with "ST_" (standard dev) and time # mdata = tbdata[col] cdata = [time, mdata] ocols = ['time', col.lower()] if not os.path.isdir(out_dir): cmd = 'mkdir ' + out_dir os.system(cmd) ofits = out_dir + col.lower()+ '_full_data_' + str(year) +'.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata) mcf.rm_file(fits)
def update_mta_comp_database(): """ updata database of mta computed msids input: none but read from /data/mta4/Deriv/*fits files outpu: updated data file: <data_dir>/<msid>_data """ # #--- get a list of data fits file names # infile = house_keeping + 'mta_comp_fits_files' data = ecf.read_file_data(infile) for fits in data: # #--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is #--- if this is not hrc (or hrc all), tail = 2 # mc = re.search('hrc', fits) if mc is not None: atemp = re.split('_', fits) btemp = re.split('.fits', atemp[1]) tail = btemp[0] else: tail = 2 [cols, tbdata] = ecf.read_fits_file(fits) time = [] for ent in tbdata.field('time'): stime = float(ent) # #--- check whether the time is in dom # if stime < 31536000: stime = ecf.dom_to_stime(float(ent)) time.append(stime) for col in cols: col = col.lower() # #--- we need only *_avg columns # mc = re.search('_avg', col) if mc is not None: vals = tbdata.field(col) ctime = [] cvals = [] for m in range(0, len(time)): # #--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999) # if str(vals[m]) in ['nan', 'NaN', 'NAN']: continue nval = float(vals[m]) if nval in [-999, -998, -99, 99, 998, 999]: continue else: ctime.append(time[m]) cvals.append(nval) atemp = re.split('_', col) msid = atemp[-2] if mcf.chkNumeric(tail): oname = msid else: oname = msid + '_' + tail print "MSID: " + str(oname) cmd = 'rm ' + data_dir + oname + '_data' os.system(cmd) # #--- read limit table for the msid # l_list = ecf.set_limit_list(msid) if len(l_list) == 0: try: l_list = mta_db[msid] except: l_list = [] update_data(msid, l_list, dset=tail, time=ctime, vals=cvals)
def update_ephhk_data(): """ update eph hk related msid data input: none output: updated data fits files """ # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- find the data period # [tstart, tstop] = find_data_period() # #--- extract fits files from archive # ofits = extract_archive_data(tstart, tstop) # #--- if no data is extracted, stop # if ofits == False: exit(1) # #--- read out the data # [cols, tbdata] = ecf.read_fits_file(ofits) cmd = 'rm -f out.fits ' + ofits os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) # #--- find useable colnames # col_list = find_col_names(cols) for col in col_list: # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, 'ephhk', dtime, data, glim)
def update_simdiag_data(date=''): """ collect sim diag msids input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # #sfile = house_keeping + 'msid_list_simdiag' sfile = './msid_list_simsupple' data = mcf.read_data_file(sfile) cols = [] g_dir = {} for ent in data: atemp = re.split('\s+', ent) cols.append(atemp[0]) g_dir[atemp[0]] = atemp[1] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() day_list = [] for year in range(1999, 2021): cyear = str(year) for mon in range(1, 13): if year == 1999: if mon < 8: continue if year == 2020: if mon > 1: break cmon = str(mon) if mon < 10: cmon = '0' + cmon if mcf.is_leapyear(year): lday = mday_list2[mon - 1] else: lday = mday_list[mon - 1] for day in range(1, lday + 1): cday = str(day) if day < 10: cday = '0' + cday sday = cyear + '-' + cmon + '-' + cday day_list.append(sday) for sday in day_list: if sday == '2020-01-17': break print("Date: " + sday) start = sday + 'T00:00:00' stop = sday + 'T23:59:59' line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = sim\n' line = line + 'level = 0\n' line = line + 'filetype = sim\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' flist = mcf.run_arc5gl_process(line) if len(flist) < 1: print("\t\tNo data") continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols_xxx, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(0, len(cols)): # #---- extract data in a list form # col = cols[k] data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # tstart = convert_time_format(start) tstop = convert_time_format(stop) update_database(msid, g_dir[msid], dtime, data, glim, pstart=tstart, pstop=tstop)
def update_eph_data_from_comm(date = ''): """ collect eph data for trending input: date ---- the data collection end date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # #sfile = house_keeping + 'eph_list' #glist = mcf.read_data_file(sfile) glist = ['ephhk',] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- find date to read the data # if date == '': yesterday = datetime.date.today() - datetime.timedelta(1) yesterday = str(yesterday).replace('-', '') date_list = create_date_list(yesterday) else: date_list = [date] error_message = '' for day in date_list: # #--- find the names of the fits files of the day of the group # dline = "Date: " + str(day) print(dline) for group in glist: print("Group: " + str(group)) cmd = 'ls /data/mta_www/mp_reports/' + day + '/' + group + '/data/* > ' + zspace os.system(cmd) tlist = mcf.read_data_file(zspace, remove=1) flist = [] for ent in tlist: mc = re.search('_STephhk_static_eio0.fits', ent) if mc is not None: flist.append(ent) # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mcf.rm_files('ztemp.fits') mfo. appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo. appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- read out the data for the full day # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(1, len(cols)): # #--- select col name without ST_ (which is standard dev) # col = cols[k] mc = re.search('ST_', col) if mc is not None: continue mc = re.search('quality', col, re.IGNORECASE) if mc is not None: continue # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = ecf.convert_unit_indicator(udict[msid]) except: tchk = 0 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # wline = uds.update_database(msid, group.capitalize(), dtime, data, glim) if wline != "": error_message = error_message + dline + '\n' + wline # #--- if there are errors, sending error message # if error_message != "": error_message = 'MTA limit trend EPH got problems: \n' + error_message fo = open(zspace, 'w') fo.write(error_message) fo.close() cmd = 'cat ' + zspace + ' | mailx -s "Subject: EPH data update problem "' cmd = cmd + '*****@*****.**' os.system(cmd) mcf.rm_files(zspace)
def update_simdiag_data(date=''): """ collect sim diag msids input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # #sfile = house_keeping + 'msid_list_simdiag' sfile = './msid_list_ephkey' data = ecf.read_file_data(sfile) cols = [] g_dir = {} for ent in data: atemp = re.split('\s+', ent) cols.append(atemp[0]) g_dir[atemp[0]] = atemp[1] # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() day_list = [] for year in range(2017, 2019): cyear = str(year) for mon in range(1, 13): if year == 2017: if mon < 7: continue if year == 2018: if mon > 1: break cmon = str(mon) if mon < 10: cmon = '0' + cmon if tcnv.isLeapYear(year) == 1: lday = mday_list2[mon - 1] else: lday = mday_list[mon - 1] for day in range(1, lday + 1): cday = str(day) if day < 10: cday = '0' + cday sday = cyear + '-' + cmon + '-' + cday day_list.append(sday) chk = 0 for sday in day_list: if sday == '2018-07-17': chk = 1 if chk == 0: continue if sday == '2018-01-21': break print "Date: " + sday start = sday + 'T00:00:00' stop = sday + 'T23:59:59' line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ephin\n' line = line + 'level = 0\n' line = line + 'filetype =ephhk\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: appendFitsTable_ascds(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): appendFitsTable_ascds('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols_xxx, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(0, len(cols)): # #---- extract data in a list form # col = cols[k] data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # tstart = convert_time_format(start) tstop = convert_time_format(stop) update_database(msid, g_dir[msid], dtime, data, glim, pstart=tstart, pstop=tstop)
def extract_data_arc5gl(detector, level, filetype, tstart, tstop, sub=''): """ extract data using arc5gl input: detector --- detector name level --- level filetype --- file name tstart --- starting time tstop --- stopping time sub --- subdetector name; defalut "" --- no sub detector output: cols --- a list of col name tdata --- a list of arrays of data """ # #--- extract ephin hk lev 0 fits data # line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ' + detector + '\n' if sub != '': line = line + 'subdetector = ' + sub + '\n' line = line + 'level = ' + level + '\n' line = line + 'filetype = ' + filetype + '\n' line = line + 'tstart = ' + str(tstart) + '\n' line = line + 'tstop = ' + str(tstop) + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" #continue return [[], []] if len(flist) < 1: print "\t\tNo data" #continue return [[], []] # #--- combined them # flen = len(flist) if flen == 0: #continue return [[], []] elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- return data # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) return [cols, tbdata]
def extract_data_arc5gl(detector, level, filetype, tstart, tstop, sub=''): """ extract data using arc5gl input: detector --- detector name level --- level filetype --- file name tstart --- starting time tstop --- stopping time sub --- subdetector name; defalut "" --- no sub detector output: cols --- a list of col name tdata --- a list of arrays of data """ # #--- extract ephin hk lev 0 fits data # line = 'operation=retrieve\n' line = line + 'dataset = flight\n' line = line + 'detector = ' + detector + '\n' if sub != '': line = line + 'subdetector = ' + sub + '\n' line = line + 'level = ' + level + '\n' line = line + 'filetype = ' + filetype + '\n' line = line + 'tstart = ' + str(tstart) + '\n' line = line + 'tstop = ' + str(tstop) + '\n' line = line + 'go\n' flist = mcf.run_arc5gl_process(line) if len(flist) < 1: print("\t\tNo data") return [[], []] # #--- combined them # flen = len(flist) if flen == 0: return [[], []] elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- return data # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) return [cols, tbdata]
def update_grad_and_comp_data(date=''): """ collect grad and comp data for trending input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used output: fits file data related to grad and comp """ # #--- read group names which need special treatment # sfile = 'grad_special_list' glist = ecf.read_file_data(sfile) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() day_list = [] for year in range(1999, 2019): cyear = str(year) for mon in range(1, 13): if year == 1999: if mon < 8: continue if year == 2018: if mon > 2: break cmon = str(mon) if mon < 10: cmon = '0' + cmon if tcnv.isLeapYear(year) == 1: lday = mday_list2[mon - 1] else: lday = mday_list[mon - 1] for day in range(1, lday + 1): cday = str(day) if day < 10: cday = '0' + cday sday = cyear + '-' + cmon + '-' + cday day_list.append(sday) for sday in day_list: print "Date: " + sday start = sday + 'T00:00:00' stop = sday + 'T23:59:59' for group in glist: print "Group: " + group line = 'operation=retrieve\n' line = line + 'dataset = mta\n' line = line + 'detector = grad\n' line = line + 'level = 0.5\n' line = line + 'filetype = ' + group + '\n' line = line + 'tstart = ' + start + '\n' line = line + 'tstop = ' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() try: cmd = ' /proj/sot/ska/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) except: cmd = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace + '> ztemp_out' os.system(cmd) mcf.rm_file(zspace) # #--- find the names of the fits files of the day of the group # try: flist = ecf.read_file_data('ztemp_out', remove=1) flist = flist[1:] except: print "\t\tNo data" continue if len(flist) < 1: print "\t\tNo data" continue # #--- combined them # flen = len(flist) if flen == 0: continue elif flen == 1: cmd = 'cp ' + flist[0] + ' ./ztemp.fits' os.system(cmd) else: mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits') if flen > 2: for k in range(2, flen): mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits') cmd = 'mv out.fits ztemp.fits' os.system(cmd) # #--- remove indivisual fits files # for ent in flist: cmd = 'rm -rf ' + ent os.system(cmd) # #--- read out the data for the full day # [cols, tbdata] = ecf.read_fits_file('ztemp.fits') cmd = 'rm -f ztemp.fits out.fits' os.system(cmd) # #--- get time data in the list form # dtime = list(tbdata.field('time')) for k in range(1, len(cols)): # #--- select col name without ST_ (which is standard dev) # col = cols[k] mc = re.search('ST_', col) if mc is not None: continue # #---- extract data in a list form # data = list(tbdata.field(col)) # #--- change col name to msid # msid = col.lower() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, data, glim)