def get_data(start, stop, year, out_dir): """ update sim flex offset data for the given data period input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 year --- data extracted year out_dir --- output_directory output: <out_dir>/Comp_save/Compsimoffset/<msid>_full_data_<year>.fits """ print str(start) + '<-->' + str(stop) for msid in ['flexadif', 'flexbdif', 'flexcdif']: if msid == 'flexadif': msid_t = '3faflaat' msid_s = '3sflxast' elif msid == 'flexbdif': msid_t = '3faflbat' msid_s = '3sflxbst' else: msid_t = '3faflcat' msid_s = '3sflxcst' out = fetch.MSID(msid_t, start, stop) tdat1 = out.vals ttime = out.times out = fetch.MSID(msid_s, start, stop) tdat2 = out.vals tlen1 = len(tdat1) tlen2 = len(tdat2) if tlen1 == 0 or tlen2 == 0: continue if tlen1 > tlen2: diff = tlen1 - tlen2 for k in range(0, diff): tdat2 = numpy.append(tdat2, tadt2[-1]) elif tlen1 < tlen2: diff = tlen2 - tlen1 for k in range(0, diff): tdat1 = numpy.append(tdat1, tadt1[-1]) ocols = ['time', msid] cdata = [ttime, tdat1 - tdat2] ofits = out_dir + msid + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata)
def get_data(start, stop, year, out_dir): """ update acis electric pwer data for a gvien period input: start --- starting time in seconds from 1998.1.1 stop --- stopping time in seconds from 1998.1.1 year --- year of the data extracted out_dir --- output directory output: <out_dir>/1dppwra_full_data_<year>.fits, <out_dir>/1dppwrb_full_data_<year>fits """ print str(start) + '<-->' + str(stop) for msid in ['1dppwra', '1dppwrb']: if msid == '1dppwra': msid_v = '1dp28avo' msid_a = '1dpicacu' else: msid_v = '1dp28bvo' msid_a = '1dpicbcu' out = fetch.MSID(msid_v, start, stop) tdat1 = out.vals ttime = out.times out = fetch.MSID(msid_a, start, stop) tdat2 = out.vals tlen1 = len(tdat1) tlen2 = len(tdat2) if tlen1 == 0 or tlen2 == 0: continue if tlen1 > tlen2: diff = tlen1 - tlen2 for k in range(0, diff): tdat2 = numpy.append(tdat2, tadt2[-1]) elif tlen1 < tlen2: diff = tlen2 - tlen1 for k in range(0, diff): tdat1 = numpy.append(tdat1, tadt1[-1]) ocols = ['time', msid] cdata = [ttime, tdat1 * tdat2] ofits = out_dir + msid + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata)
def run_update_with_ska(msid, group, msid_sub_list=[], glim=''): """ extract data from ska database and update the data for the msids in the msid_list input: msid --- a list of msids group --- the group of the msids msid_sub_list --- a list of lists of: [msid, msid_1, msid_2, operand] this is used to compute the first msid from following two msid values with operand (+/-/*) glim --- glim usually found in this function, but you can give it; default: '' output: <msid>_data.fits, <msid>_short_data,fits, <msid>_week_data.fits """ # #--- get basic information dict/list # [udict, ddict, mta_db, mta_cross] = ecf.get_basic_info_dict() # #--- set starting and stopping data period # test_fits = data_dir + group.capitalize() + '/' + msid + '_data.fits' if os.path.isfile(test_fits): tstart = ecf.find_the_last_entry_time(test_fits) ttemp = time.strftime("%Y:%j:00:00:00", time.gmtime()) tstop = Chandra.Time.DateTime(ttemp).secs - 86400.0 if tstop < tstart: exit(1) if len(msid_sub_list) != 0: [dtime, tdata] = compute_sub_msid(msid, msid_sub_list, tstart, tstop) else: out = fetch.MSID(msid, tstart, tstop) ok = ~out.bads dtime = out.times[ok] tdata = out.vals[ok] # #--- fetch occasionally adds -999.xxx to the output data of some msids; remove them (Jun 13, 2018) # tind = [(tdata > -999) | (tdata <= -1000)] dtime = dtime[tind] tdata = tdata[tind] # #--- get limit data table for the msid # if glim == '': try: tchk = ecf.convert_unit_indicator(udict[msid]) except: tchk = 0 if msid in sp_limt_case_c: tchk = 1 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # update_database(msid, group, dtime, tdata, glim)
def get_data(start, stop, year, msid_list, out_dir): """ update msid data in msid_list for the given data period input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 year --- the year in which data is extracted msid_list --- a list of msids out_dir --- output_directory """ print(str(start) + '<-->' + str(stop)) for msid in msid_list: out = fetch.MSID(msid, start, stop) tdat = out.vals - 273.15 ttime = out.times ocols = ['time', msid] cdata = [ttime, tdat] ofits = out_dir + msid + '_full_data_' + str(year) +'.fits' if os.path.isfile(ofits): ecf.update_fits_file(ofits, ocols, cdata) else: ecf.create_fits_file(ofits, ocols, cdata)
def compute_sub_msid(msid, msid_sub_list, tstart, tstop): """ return computed msid's data input: msid --- msid msid_sub_list ---a list of lists of: [msid, msid_1, msid_2, operand] this is used to compute the first msid from following two msid values with operand (+/-/*) tstart --- start time in seconds from 1998.1.1 tstop --- stop tim in seconds from 1998.1.1 output: ttime --- array of time values tdata --- array of the data values """ chk = 0 for m in range(0, len(msid_sub_list)): if msid == msid_sub_list[m][0]: msid_t = msid_sub_list[m][1] msid_s = msid_sub_list[m][2] opr = msid_sub_list[m][3] chk = 1 break if chk > 0: out = fetch.MSID(msid_t, tstart, tstop) tdata_t = out.vals ttime = out.times out = fetch.MSID(msid_s, tstart, tstop) tdata_s = out.vals if opr == '-': tdata = tdata_t - tdata_s elif opr == '*': tdata = tdata_t * tdata_s else: tdata = tdata_t + tdata_s else: try: out = fetch.MSID(msid, tstart, tstop) tdata = out.vals ttime = out.times except: tdata = [] ttime = [] return [ttime, tdata]
def get_data(start, stop, year): print str(start) + '<-->' + str(stop) for msid in ['flexadif', 'flexbdif', 'flexcdif']: if msid == 'flexadif': msid_t = '3faflaat' msid_s = '3sflxast' elif msid == 'flexbdif': msid_t = '3faflbat' msid_s = '3sflxbst' else: msid_t = '3faflcat' msid_s = '3sflxcst' out = fetch.MSID(msid_t, start, stop) tdat1 = out.vals ttime = out.times out = fetch.MSID(msid_s, start, stop) tdat2 = out.vals tlen1 = len(tdat1) tlen2 = len(tdat2) if tlen1 == 0 or tlen2 == 0: continue if tlen1 > tlen2: diff = tlen1 - tlen2 for k in range(0, diff): tdat2 = numpy.append(tdat2, tadt2[-1]) elif tlen1 < tlen2: diff = tlen2 - tlen1 for k in range(0, diff): tdat1 = numpy.append(tdat1, tadt1[-1]) ocols = ['time', msid] cdata = [ttime, tdat1 - tdat2] ofits = dout + msid + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata)
def get_data(start, stop, year): print str(start) + '<-->' + str(stop) for msid in ['1dppwra', '1dppwrb']: if msid == '1dppwra': msid_v = '1dp28avo' msid_a = '1dpicacu' else: msid_v = '1dp28bvo' msid_a = '1dpicbcu' out = fetch.MSID(msid_v, start, stop) tdat1 = out.vals ttime = out.times out = fetch.MSID(msid_a, start, stop) tdat2 = out.vals tlen1 = len(tdat1) tlen2 = len(tdat2) if tlen1 == 0 or tlen2 == 0: continue if tlen1 > tlen2: diff = tlen1 - tlen2 for k in range(0, diff): tdat2 = numpy.append(tdat2, tadt2[-1]) elif tlen1 < tlen2: diff = tlen2 - tlen1 for k in range(0, diff): tdat1 = numpy.append(tdat1, tadt1[-1]) ocols = ['time', msid] cdata = [ttime, tdat1 * tdat2] ofits = dout + msid + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata)
def fetch_eng_data(msid, start, stop): """ get eng data from archieve input: msid --- msid start --- start time in sec from 1998.1.1 stop --- stop time in sec from 1988.1.1 output: [time, data] --- a list of time array and data array """ # #--- read data from database # out = fetch.MSID(msid, start, stop) # #--- collect data in 5 min intervals and take an average # stime = [] data = [] tsave = [] pstart = start pstop = pstart + 300.0 for k in range(0, len(out.times)): if out.times[k] < pstart: continue # #--- collected data for the last 5 mins # elif out.times[k] > pstop: stime.append(pstart + 150.0) # #--- if no data, just put 0.0 # if len(tsave) == 0: data.append(0.0) # #--- take an avearge # else: data.append(numpy.mean(tsave)) tsave = [] pstart = pstop pstop = pstart + 300.0 else: tsave.append(out.vals[k]) # #--- convert the list into an array form before returning # atime = numpy.array(stime) adata = numpy.array(data) return [atime, adata]
def get_data(msid, start, stop): """ create an interactive html page for a given msid input: msid --- msid oup --- group name start --- start time stop --- stop time output: ttime --- a list of time data tdata --- a list of data """ # start = ecf.check_time_format(start) # stop = ecf.check_time_format(stop) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- get limit data table for the msid # try: uck = udict[msid] if uck.lower() == 'k': tchk = 1 else: tchk = ecf.convert_unit_indicator(uchk) except: tchk = 0 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- extract data from archive # chk = 0 try: out = fetch.MSID(msid, start, stop) tdata = out.vals ttime = out.times except: tdata = [] ttime = [] return [ttime, tdata]
def get_data_from_ska(msid, tstart, tstop): """ extract data from ska database input: msid --- msid tstart --- starting time in seconds from 1998.1.1 tstop --- stopping time in seconds from 1998.1.1 output: time --- a list of time data --- a list of data """ out = fetch.MSID(msid, tstart, tstop) time = out.times data = out.vals return [time, data]
def get_temp_data(msid, xdata): """ get temperature data of msid for the given time spots in the list input: msid --- msid xdata --- a list of time data in seconds from 1998.1.1 output: temperature --- a list of temperature corresponding to the time list """ temperature = [] for m in range(0, len(xdata)): start = xdata[m] - 60.0 stop = xdata[m] + 60.0 out = fetch.MSID(msid, start, stop) val = numpy.mean(out.vals) temperature.append(val) return temperature
def fetchTelemetry(msid, start, now, filter_bad=True): ''' Fetch the telemetry from the Ska archive for the given MSID. filter_bad will call fetch.py's option to automatically filter bad data. The stat call should be None, as we don't want daily statistics. Returns an instance of class MSID ''' telemetry = fetch.MSID(msid, start, now, filter_bad=filter_bad, stat='daily') return telemetry
def read_data(tstart, tstop): """ extract needed data from sot database input: tstart --- starting time in seconds from 1998.1.1 tstop --- stopping time in seconds from 1998.1.1 output: data --- a list of arrays of data """ save = [] for msid in bias_list: out = fetch.MSID(msid, tstart, tstop) val = out.vals save.append(val) data = [out.times] + save return data
def get_data(start, stop, year, acis_list): print str(start) + '<-->' + str(stop) for msid in acis_list: out = fetch.MSID(msid, start, stop) tdat = out.vals - 273.15 ttime = out.times ocols = ['time', msid] cdata = [ttime, tdat] ofits = out_dir + msid + '_full_data_' + str(year) + '.fits' if os.path.isfile(ofits): update_fits_file(ofits, ocols, cdata) else: create_fits_file(ofits, ocols, cdata)
def create_interactive_page(msid, group, start, stop, step): """ create an interactive html page for a given msid input: msid --- msid start --- start time stop --- stop time step --- bin size in seconds """ start = check_time_format(start) stop = check_time_format(stop) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- extract needed data and save in fits file # try: out = fetch.MSID(msid, '2017:001:00:00:00', '2017:002') except: missed = house_keeping + '/missing_data' fo = open(missed, 'a') fo.write(msid) fo.write('\n') fo.close() fits_data = update_database(msid, group, glim, start, stop, step)
def fetch_eng_data(msid_list, start, stop): """ get eng data from archieve input: msid_list --- a list of msids start --- start time in <yyyy>:<ddd>:<hh>:<mm>:<ss> stop --- stop time in <yyyy>:<ddd>:<hh>:<mm>:<ss> output: results --- a list of lists of data extracted including time at beginning """ results = [] chk = 0 for msid in msid_list: out = fetch.MSID(msid, start, stop) if chk == 0: time = list(out.times) results.append(time) chk = 1 data = list(out.vals) results.append(data) return results
sys.path.insert(0, '..') if not 'ENG_ARCHIVE' in os.environ: os.environ['ENG_ARCHIVE'] = os.path.abspath(os.getcwd() + '/eng_archive') outfile = opt.out + '.test' else: outfile = opt.out + '.flight' import Ska.engarchive.fetch as fetch print 'Fetch file is', fetch.__file__ print 'ENG_ARCHIVE is', os.environ.get('ENG_ARCHIVE') msids = ('1crat', 'fptemp_11', 'orbitephem0_x', 'sim_z', 'tephin', 'cvcductr', 'dp_dpa_power') attrs = ('times', 'vals', 'quals', 'stds', 'mins', 'maxes', 'means', 'p01s', 'p05s', 'p16s', 'p50s', 'p84s', 'p95s', 'p99s') out = dict() for msid in msids: print 'Getting', msid dat = fetch.MSID(msid, opt.start, opt.stop) dat5 = fetch.MSID(msid, opt.start, opt.stop, stat='5min') datd = fetch.MSID(msid, opt.start, opt.stop, stat='daily') out[msid] = dict(dat=dict((x, getattr(dat, x)) for x in attrs if hasattr(dat, x)), dat5=dict((x, getattr(dat5, x)) for x in attrs if hasattr(dat5, x)), datd=dict((x, getattr(datd, x)) for x in attrs if hasattr(datd, x))) out['ENG_ARCHIVE'] = os.environ.get('ENG_ARCHIVE') out['file'] = fetch.__file__ pickle.dump(out, open(outfile, 'w'), protocol=-1)
def run_condtion_msid(msid, start, stop, period, alimit, cnd_msid): """ extract data from ska database and analyze data input: msid --- msid start --- starting time in seconds from 1998.1.1 stop --- stopping time in seconds from 1998.1.1 period --- data collection interval in seconds (e.g. 300, 3600, or 86400) alimit --- a list of lists of limits cnd_msid ---- msid which tells which limit set to use for given time output: save --- a list of list of data: time, average, median, std, min, max, ratio of yellow lower violation, ratio of yellow upper violation, ratio of rd lower violation, ratio of red upper violation, total data in the period, yellow lower limit, yellow upper limit, red lower limit, red upper limit state """ # #--- extract data with ska fetch for the given time period # out = fetch.MSID(msid, start, stop) ok = ~out.bads dtime = out.times[ok] if len(dtime) < 1: return [] tdata = out.vals[ok] tmax = dtime[-1] # #--- for the case this is multi limit case # if cnd_msid != 'none': out = fetch.MSID(cnd_msid, start, stop) mtime = out.times mdata = out.vals mlen = len(mdata) # #--- for the case this is single limit case # else: mdata = ['none'] * len(dtime) # #--- there are 15 elements to keep in the output data # save = [] for k in range(0, 16): save.append([]) # #--- compute how many data collection periods exist for a given data period # n_period = int((stop - start) / period) + 1 # #--- collect data in each time period and compute statistics # for k in range(0, n_period): begin = start + k * period end = begin + period ctime = begin + 0.5 * period # #--- find the state of condition msid for this period of time # if cnd_msid == 'none': mkey = 'none' else: pos = int(mlen * begin / tmax) - 1 if pos < 0: pos = 0 if pos >= mlen: pos = mlen - 1 mkey = mdata[pos].lower() # #--- set limit range only once at the beginning of each data collection period # try: limit_table = find_limits(begin, mkey, alimit) [y_low, y_top, r_low, r_top] = limit_table except: limit_table = [-9999998.0, 9999998.0, -9999999.0, 9999999.0] [y_low, y_top, r_low, r_top] = [-9999998.0, 9999998.0, -9999999.0, 9999999.0] # #--- select data between the period # ind = dtime >= begin btemp = dtime[ind] sdata = tdata[ind] ind = btemp < end sdata = sdata[ind] dcnt = len(sdata) if dcnt < 1: continue # #--- get stats # dmin = min(sdata) dmax = max(sdata) avg = numpy.mean(sdata) # #--- if the value is too large something is wrong: so skip it # if abs(avg) > 100000000.0: continue med = numpy.median(sdata) std = numpy.std(sdata) # #--- count number of violations # [y_lc, y_uc, r_lc, r_uc] = find_limit_violatons(sdata, limit_table) # #--- save the resuts # save[0].append(float(int(ctime))) save[1].append(float("%3.2f" % avg)) save[2].append(float("%3.2f" % med)) save[3].append(float("%3.2f" % std)) save[4].append(float("%3.2f" % dmin)) save[5].append(float("%3.2f" % dmax)) save[6].append(float("%1.3f" % (y_lc / dcnt))) save[7].append(float("%1.3f" % (y_uc / dcnt))) save[8].append(float("%1.3f" % (r_lc / dcnt))) save[9].append(float("%1.3f" % (r_uc / dcnt))) save[10].append(dcnt) save[11].append(float("%3.2f" % y_low)) save[12].append(float("%3.2f" % y_top)) save[13].append(float("%3.2f" % r_low)) save[14].append(float("%3.2f" % r_top)) save[15].append(mkey) return save
def get_acen_data(tstart, tstop, file_id, fid_detect): """ for given fid light information, extract acen fits files and extract needed information input: tstart --- starting time tstop --- stopping time file_id --- id of the acen fits file fid_detect --- a dictionary of information [<slot id>, <id string> <id number>] output: udated data file, e.g., I-1, S-3, H-I-1, H-S-3 etc in <data_dir> """ # #--- first just get a list of potential acent fits files # acent_list = call_arc5gl('browse', 'pcad', 1, tstart=tstart, tstop=tstop,\ filetype='acacent', sub='aca') # #--- compare the list with a file id, and if it is found, procceed farther # for fname in file_id: chk = 0 for comp in acent_list: mc = re.search(fname, comp) if mc is not None: filename = comp chk = 1 break if chk == 0: continue # #--- extract an acen fits file # [fits] = call_arc5gl('retrieve', 'pcad', 1, tstart='', tstop='',\ filetype='acacent', filename=filename, sub='aca') ff = pyfits.open(fits) data = ff[1].data ff.close() mcf.rm_file(fits) # #--- extract needed information for each slot # for m in range(0, len(fid_detect[fname][0])): slot_id = fid_detect[fname][0][m] mask = data['slot'] == slot_id out = data[mask] time = out['time'] cent_i = out['cent_i'] cent_j = out['cent_j'] ang_y = out['ang_y'] ang_z = out['ang_z'] alg = out['alg'] if len(time) == 0: continue # #---- take 5 min average for the data # try: begin = time[0] except: continue end = begin + 300.0 m = 0 k_list = [] sline = '' for k in range(m, len(time)): if time[k] < begin: continue elif time[k] >= begin and time[k] < end: k_list.append(k) else: try: atime = numpy.mean(time[k_list[0]:k_list[-1]]) acent_i = numpy.mean(cent_i[k_list[0]:k_list[-1]]) acent_j = numpy.mean(cent_j[k_list[0]:k_list[-1]]) aang_y = numpy.mean(ang_y[k_list[0]:k_list[-1]]) aang_z = numpy.mean(ang_z[k_list[0]:k_list[-1]]) aalg = alg[k_list[-1]] except: continue # #--- find fapos and tscpos info near to the given time interval # flist = fetch.MSID('3fapos', time[k_list[0]], time[k_list[-1]]) tslist = fetch.MSID('3tscpos', time[k_list[0]], time[k_list[-1]]) fapos = numpy.mean(flist.vals) tscpos = numpy.mean(tslist.vals) sline = sline + str(atime) + '\t' sline = sline + str(fid_detect[fname][0][m]) + '\t' sline = sline + str(fid_detect[fname][2][m]) + '\t' sline = sline + str(aalg) + '\t' sline = sline + str(format(acent_i, '.3f')) + '\t' sline = sline + str(format(acent_j, '.3f')) + '\t' sline = sline + str(format(aang_y, '.6f')) + '\t' sline = sline + str(format(aang_z, '.6f')) + '\t' sline = sline + str(fapos) + '\t' sline = sline + str(tscpos) + '\n' k_list = [] begin = end end += 300.0 # #--- write out the results # ofile = data_dir + fid_detect[fname][1][m] with open(ofile, 'a') as fo: fo.write(sline)
def get_data(start, stop, year, out_dir): """ extract data and update the compgradkodak related data sets for the given period input: start --- start time in seconds from 1998.1.1 stop --- stop time in seconds from 1998.1.1 out_dir --- output_directory output: <out_dir>/<msid>_fill_data_<year>.fits """ empty = [0] # #--- extract 4rt*** data # rt7 = [] for k in range(0, 12): if k < 10: msid = '4rt70' + str(k) + 't' else: msid = '4rt7' + str(k) + 't' try: out = fetch.MSID(msid, start, stop) data = out.vals ttime = out.times tlist = list(ttime) rt7.append(data) except: rt7.append(empty) # #--- extract 4rt575t separately # out = fetch.MSID('4rt575t', start, stop) rt575 = out.vals # #--- create empty array and initialize ohrthr and oobthr lists # tlen = len(ttime) empty = numpy.zeros(tlen) ohrthr = [empty] oobthr = [empty] # #--- fill them up # for k in range(1, 65): if k < 10: msid = 'ohrthr0' + str(k) else: msid = 'ohrthr' + str(k) try: out = fetch.MSID(msid, start, stop) data = out.vals otime = out.times # #--- since 4rt arrays are 36 time dense, match the ohrthr and oobtrhr #--- by filling the gaps between # adata = fill_gaps(ttime, otime, data) ohrthr.append(adata) except: ohrthr.append(empty) if k < 10: msid = 'oobthr0' + str(k) else: msid = 'oobthr' + str(k) try: out = fetch.MSID(msid, start, stop) data = out.vals otime = out.times adata = fill_gaps(ttime, otime, data) oobthr.append(adata) except: oobthr.append(empty) # #--- now compute each quantity for the given time period # hrmaavg = [] hrmacav = [] hrmaxgrd = [] hrmaradgrd = [] obaavg = [] obaconeavg = [] fwblkhdt = [] aftblkhdt = [] obaaxgrd = [] mzobacone = [] pzobacone = [] obadiagrad = [] hrmarange = [] tfterange = [] hrmastrutrnge = [] scstrutrnge = [] # #--- save time stamp separately for each data # t_hrmaavg = [] t_hrmacav = [] t_hrmaxgrd = [] t_hrmaradgrd = [] t_obaavg = [] t_obaconeavg = [] t_fwblkhdt = [] t_aftblkhdt = [] t_obaaxgrd = [] t_mzobacone = [] t_pzobacone = [] t_obadiagrad = [] t_hrmarange = [] t_tfterange = [] t_hrmastrutrnge = [] t_scstrutrnge = [] for k in range(0, tlen): out = compute_hrmaavg(ohrthr, k) if out != 'na': hrmaavg.append(out) t_hrmaavg.append(tlist[k]) #------------------------- out = compute_hrmacav(ohrthr, k) if out != 'na': hrmacav.append(out) t_hrmacav.append(tlist[k]) #------------------------- out = compute_hrmaxgrd(ohrthr, k) if out != 'na': hrmaxgrd.append(out) t_hrmaxgrd.append(tlist[k]) #------------------------ out = compute_hrmaradgrd(ohrthr, k) if out != 'na': hrmaradgrd.append(out) t_hrmaradgrd.append(tlist[k]) #------------------------ out = compute_obaavg(oobthr, k) if out != 'na': obaavg.append(out) t_obaavg.append(tlist[k]) #------------------------ out = compute_obaconeavg(oobthr, k) if out != 'na': obaconeavg.append(out) t_obaconeavg.append(tlist[k]) #------------------------ out = compute_fwblkhdt(oobthr, rt7, k) chk1 = 0 if out != 'na': fwblkhdt.append(out) t_fwblkhdt.append(tlist[k]) chk1 = 1 #------------------------ out = compute_aftblkhdt(oobthr, k) chk2 = 0 if out != 'na': aftblkhdt.append(out) t_aftblkhdt.append(tlist[k]) chk2 = 1 #------------------------ if (chk1 == 1) and (chk2 == 1): out = compute_obaaxgrd(fwblkhdt[-1], aftblkhdt[-1]) if out != 'na': obaaxgrd.append(out) t_obaaxgrd.append(tlist[k]) #------------------------ out = compute_mzobacone(oobthr, rt575, k) chk1 = 0 if out != 'na': mzobacone.append(out) t_mzobacone.append(tlist[k]) chk1 = 1 #------------------------ out = compute_pzobacone(oobthr, k) chk2 = 0 if out != 'na': pzobacone.append(out) t_pzobacone.append(tlist[k]) chk2 = 1 #------------------------ if (chk1 == 1) and (chk2 == 1): out = compute_obadiagrad(mzobacone[-1], pzobacone[-1]) if out != 'na': obadiagrad.append(out) t_obadiagrad.append(tlist[k]) #------------------------ out = compute_hrmarange(ohrthr, k) if out != 'na': hrmarange.append(out) t_hrmarange.append(tlist[k]) #------------------------ out = compute_tfterange(oobthr, k) if out != 'na': tfterange.append(out) t_tfterange.append(tlist[k]) #------------------------ out = compute_hrmastrutrnge(oobthr, k) if out != 'na': hrmastrutrnge.append(out) t_hrmastrutrnge.append(tlist[k]) #------------------------ out = compute_scstrutrnge(oobthr, k) if out != 'na': scstrutrnge.append(out) t_scstrutrnge.append(tlist[k]) # #--- now create/update output fits files # for col in ['hrmaavg', 'hrmacav', 'hrmaxgrd', 'hrmaradgrd', 'obaavg', 'obaconeavg', 'fwblkhdt',\ 'aftblkhdt', 'obaaxgrd', 'mzobacone', 'pzobacone', 'obadiagrad', 'hrmarange',\ 'tfterange', 'hrmastrutrnge', 'scstrutrnge']: exec "odata = %s" % (col) exec "tdata = t_%s" % (col) olen = len(odata) tdata = numpy.array(tdata) odata = numpy.array(odata) cdata = [tdata, odata] cols = ['time', col] fits = out_dir + col + '_full_data_' + str(year) + '.fits' if os.path.isfile(fits): ecf.update_fits_file(fits, cols, cdata) else: ecf.create_fits_file(fits, cols, cdata)
def check_condition(msid, tstart, tstop, condition, achk=0): """ find the periods of the time when the condition is met input: msid --- msid tstart --- starting time in seconds from 1998.1.1 tstop --- stopping time in seconds from 1998.1.1 condition --- condition value. if the value is larger than this it is "ON" period achk --- indicator of whether we need data cleaning; achk=1: yes output: c_start --- a list of starting time of the active period c_stop --- a list of stopping time of the active period """ #print msid + '<-->' + str(tstart) + '<-->' + str(tstop) out = fetch.MSID(msid, tstart, tstop) cptime = out.times cvals = out.vals # #--- if voltage cases (achk ==1), use only data in the ceratin range # if achk == 1: atime = [] avals = [] for k in range(0, len(cvals)): if cvals[k] > 200: continue if cvals[k] > 5 and cvals[k] < 80: continue atime.append(cptime[k]) avals.append(cvals[k]) # #--- non voltage case # else: atime = cptime avals = cvals # #--- find start and stop time of the period where the condtion is met #--- if the value is larger than condition, it is "ON". # c_start = [] c_stop = [] chk = 0 for k in range(0, len(atime)): if chk == 0 and avals[k] > condition: c_start.append(atime[k]) chk = 1 elif chk == 1 and avals[k] < condition: c_stop.append(atime[k]) chk = 0 else: continue # #--- if the period is not closed, use the last entry to close it # if chk == 1: c_stop.append(atime[-1]) return [c_start, c_stop]
## The basic process of fetching data always starts with importing the module ## into the python session:: # Licensed under a 3-clause BSD style license - see LICENSE.rst print "Welcome to the fetch module!" import Ska.engarchive.fetch as fetch # <demo> --- stop --- ## The ``as fetch`` part of the ``import`` statement just creates an short alias ## to avoid always typing the somewhat lengthy ``Ska.engarchive.fetch.MSID(..)``. ## Fetching and plotting full time-resolution data for a single MSID is then quite ## easy:: tephin = fetch.MSID('tephin', '2009:001', '2009:007') # (MSID, start, stop) clf() plot(tephin.times, tephin.vals) # <demo> --- stop --- ## The ``tephin`` variable returned by ``fetch.MSID()`` is an ``MSID`` object and ## we can access the various object attributes with ``<object>.<attr>``. The ## timestamps ``tephin.times`` and the telemetry values ``tephin.vals`` are both ## numpy arrays. As such you can inspect them and perform numpy operations and ## explore their methods:: print type(tephin) print type(tephin.vals) print tephin.vals.mean() print tephin.vals.min() print tephin.times[1:20] print tephin.vals[1:20]
#!/usr/bin/env /data/mta/Script/Python3.6/envs/ska3/bin/python import os import sys import re import string import time import numpy import astropy.io.fits as pyfits import Ska.engarchive.fetch as fetch import Chandra.Time if len(sys.argv) > 1: msid = sys.argv[1] else: print("msid??") try: #out = fetch.MSID(msid, '2017:001:00:00:00', '2017:002') out = fetch.MSID(msid, '2000:060:00:00:00', '2000:062:00:00:00') tdata = out.vals print("I AM HERE: " + str(tdata)) except: print("msid is not in the database")
def find_cold_plates(t_list): """ create cold plate temperature data lists corresponding to the given time list input: t_list --- a list of time output: crat --- a list of temperature of plate A crbt --- a list of temperature of plate B """ # #--- set data time interval # start = t_list[0] stop = t_list[-1] # #--- cold plate A # out = fetch.MSID('1crat', start, stop) tlist = out.times alist = out.vals # #--- cold plate B # out = fetch.MSID('1crbt', start, stop) blist = out.vals # #--- make sure that data have the same numbers of entries # alen = len(alist) blen = len(blist) if alen < blen: blist = blist[:alen] elif alen > blen: for k in range(blen, alen): blist.append(blist[-1]) # #--- find the cold plate temperatures correpond to the given time # crat = [] crbt = [] for tent in t_list: # #-- take +/- 10 seconds # begin = tent - 30 end = tent + 30 m = 0 chk = 0 for k in range(m, alen): if (tlist[k] >= begin) and (tlist[k] <= end): crat.append(float(alist[k]) - 273.15) crbt.append(float(blist[k]) - 273.15) m = k - 10 if m < 0: m = 0 chk = 1 break if chk == 0: crat.append(999.0) crbt.append(999.0) return [crat, crbt]
def create_interactive_page(msid, group, start, stop, step): """ create an interactive html page for a given msid input: msid --- msid group --- group name start --- start time stop --- stop time step --- bin size in seconds """ start = ecf.check_time_format(start) stop = ecf.check_time_format(stop) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- get limit data table for the msid # try: tchk = ecf.convert_unit_indicator(udict[msid]) except: tchk = 0 glim = ecf.get_limit(msid, tchk, mta_db, mta_cross) # #--- extract data from archive # chk = 0 try: out = fetch.MSID(msid, start, stop) tdata = out.vals ttime = out.times except: # #--- if no data in archive, try mta local database # try: [ttime, tdata] = uds.get_mta_fits_data(msid, start, stop) # #--- if it is also failed, return the empty data set # except: chk = 1 # #--- only short_p can change step size (by setting "step") # if chk == 0: [week_p, short_p, long_p] = uds.process_day_data(msid, ttime, tdata, glim, step=step) # #--- try to find data from ska or mta local data base # try: fits_data = create_inter_fits(msid, short_p) # #--- for the case, the data is mta special cases # except: fits_data = 'na' else: fits_data = 'na' # #--- create interactive html page # create_html_page(msid, fits_data, step) # #--- remove fits file # if fits_data != 'na': cmd = 'rm -rf ' + fits_data os.system(cmd)
def get_data_from_archive(msid, start, stop, glim, step=300.0): """ extract data from the archive and compute the stats input: msid --- msid of the data start --- start time stop --- stop time glim --- a list of limit tables step --- interval of the data. defalut: 3600 sec output: a list of two lists which contain: week_p: wtime --- a list of time in sec from 1998.1.1 wdata --- a list of the mean of each interval wmed --- a list of the median of each interval wstd --- a list of the std of each interval wmin --- a list of the min of each interval wmax --- a list of the max of each interval wyl --- a list of the rate of yellow lower violation wyu --- a list of the rate of yellow upper violation wrl --- a list of the rate of red lower violation wru --- a list of the rate of red upper violation wcnt --- a list of the total data counts wyl --- a list of the lower yellow limits wyu --- a list of the upper yellow limits wrl --- a list of the lower red limits wru --- a list of the upper red limits """ # #--- 5 min step data # wtime = [] wdata = [] wmed = [] wstd = [] wmin = [] wmax = [] wyl = [] wyu = [] wrl = [] wru = [] wcnt = [] wsave = [] try: out = fetch.MSID(msid, start, stop) tdata = out.vals ttime = out.times data = [] dtime = [] for k in range(0, len(tdata)): try: test = int(float(tdata[k])) except: continue if int(abs(tdata[k])) == 999: continue data.append(tdata[k]) dtime.append(ttime[k]) data = numpy.array(data) dtime = numpy.array(dtime) spos = 0 chk = 1 send = dtime[spos] + step for k in range(0, len(dtime)): if dtime[k] < send: chk = 0 else: sdata = data[spos:k] avg = sdata.mean() med = numpy.median(sdata) sig = sdata.std() amin = sdata.min() amax = sdata.max() ftime = dtime[spos + int(0.5 * (k - spos))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(sdata, vlimits) wtime.append(ftime) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) spos = k send = dtime[k] + step chk = 1 # #--- check whether there are any left over; if so add it to the data lists # if chk == 0: rdata = data[spos:k] avg = rdata.mean() med = numpy.median(rdata) sig = rdata.std() amin = rdata.min() amax = rdata.max() ftime = dtime[spos + int(0.5 * (k - spos))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(rdata, vlimits) wtime.append(dtime[spos + int(0.5 * (k - spos))]) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) except: pass week_p = [wtime, wdata, wmed, wstd, wmin, wmax, wyl, wyu, wrl, wru, wcnt] # #--- adding limits to the table # vtemp = [[], [], [], []] for k in range(0, len(wsave)): for m in range(0, 4): vtemp[m].append(wsave[k][m]) week_p = week_p + vtemp return [week_p]
def get_data_from_archive(msid, start, stop, glim, step): """ extract data from the archive and compute the stats input: msid --- msid of the data start --- start time stop --- stop time glim --- a list of limit tables step --- a bin size in seconds output: a list of two lists which contain: week_p: wtime --- a list of time in sec from 1998.1.1 wdata --- a list of the mean of each interval wmed --- a list of the median of each interval wstd --- a list of the std of each interval wmin --- a list of the min of each interval wmax --- a list of the max of each interval wyl --- a list of the rate of yellow lower violation wyu --- a list of the rate of yellow upper violation wrl --- a list of the rate of red lower violation wru --- a list of the rate of red upper violation wcnt --- a list of the total data counts wyl --- a list of the lower yellow limits wyu --- a list of the upper yellow limits wrl --- a list of the lower red limits wru --- a list of the upper red limits """ # # wtime = [] wdata = [] wmed = [] wstd = [] wmin = [] wmax = [] wyl = [] wyu = [] wrl = [] wru = [] wcnt = [] wsave = [] vsave = [] # #--- extract data from archive # xxx = 9999 if xxx == 9999: ###try: out = fetch.MSID(msid, start, stop) tdata = out.vals ttime = out.times data = [] dtime = [] # #--- if the data is not given, the database desplay it as -999.999 (or similar); so drop them # for k in range(0, len(tdata)): try: test = int(float(tdata[k])) except: continue if int(abs(tdata[k])) == 999: continue data.append(tdata[k]) dtime.append(ttime[k]) data = numpy.array(data) dtime = numpy.array(dtime) # #--- if a full resolution is asked... # if step == 0.0: wtime = dtime wdata = data wmed = data wstd = [0] * len(data) wmin = data wmax = data for m in range(0, len(dtime)): vlimits = find_violation_range(glim, dtime[m]) darray = numpy.array([data[m]]) [yl, yu, rl, ru, tot] = find_violation_rate(darray, vlimits) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(1) wsave.append(vlimits) # #--- if asked, devide the data into a smaller period (step size) # else: spos = 0 spos2 = 0 chk = 1 chk2 = 2 send2 = dtime[spos2] + step for k in range(0, len(dtime)): if dtime[k] < send2: chk2 = 0 else: sdata = data[spos2:k] if len(sdata) <= 0: spos2 = k send2 = dtime[k] + step chk2 = 1 continue avg = sdata.mean() med = numpy.median(sdata) sig = sdata.std() amin = sdata.min() amax = sdata.max() ftime = dtime[spos2 + int(0.5 * (k - spos2))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(sdata, vlimits) wtime.append(ftime) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) spos2 = k send2 = dtime[k] + step chk2 = 1 # #--- check whether there are any left over; if so add it to the data lists # if chk2 == 0: rdata = data[spos2:k] if len(rdata) > 0: avg = rdata.mean() med = numpy.median(rdata) sig = rdata.std() amin = rdata.min() amax = rdata.max() ftime = dtime[spos2 + int(0.5 * (k - spos2))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(rdata, vlimits) wtime.append(dtime[spos2 + int(0.5 * (k - spos2))]) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) else: #----REMOVE!! ###except: pass week_p = [wtime, wdata, wmed, wstd, wmin, wmax, wyl, wyu, wrl, wru, wcnt] # #--- adding limits to the table # vtemp = [[], [], [], []] for k in range(0, len(wsave)): for m in range(0, 4): vtemp[m].append(wsave[k][m]) week_p = week_p + vtemp return week_p
def tephin_leak_data_update(year=''): """ update tephin - ephin rate/leak current data input: year --- year of the data to be updated. if it is '', the current year is used output: <data_dir>/<msid>/<msid>_data_<year>.fits """ # #--- set data extraction period # tout = set_time_period(year) if len(tout) == 6: [lstart, lstop, lyear, tstart, tstop, year] = tout chk = 1 else: [tstart, tstop, year] = tout chk = 0 # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = read_cross_check_table() # #--- extract tephin data # tchk = convert_unit_indicator(udict['tephin']) glim = get_limit('tephin', tchk, mta_db, mta_cross) # #--- for the case the time span goes over the year boundary # if chk == 1: ltephin = update_database('tephin', 'Eleak', glim, ltstart, ltstop, lyear) tephin = update_database('tephin', 'Eleak', glim, tstart, tstop, year) # #--- read msid list # mfile = house_keeping + 'msid_list_eph_tephin' data = ecf.read_file_data(mfile) for ent in data: # #--- find msid and group name # mc = re.search('#', ent) if mc is not None: continue try: [msid, group] = re.split('\s+', ent) except: atemp = re.split('\s+', ent) msid = atemp[0] group = atemp[1] msid.strip() group.strip() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 glim = get_limit(msid, tchk, mta_db, mta_cross) # #--- update database # try: out = fetch.MSID(msid, '2017:001:00:00:00', '2017:002') print "MSID: " + msid except: missed = house_keeping + '/missing_data' fo = open(missed, 'a') fo.write(msid) fo.write('\n') fo.close() continue # #--- for the case, the time span goes over the year boundary # if chk == 1: update_database(msid, group, glim, ltstart, ltstop, lyear, sdata=ltephin) update_database(msid, group, glim, tstart, tstop, year, sdata=tephin)
def get_data_from_archive(msid, start, stop, glim, step=3600.0): """ extract data from the archive and compute the stats input: msid --- msid of the data start --- start time stop --- stop time glim --- a list of limit tables step --- interval of the data. defalut: 3600 sec output: a list of two lists which contain: week_p: wtime --- a list of time in sec from 1998.1.1 wdata --- a list of the mean of each interval wmed --- a list of the median of each interval wstd --- a list of the std of each interval wmin --- a list of the min of each interval wmax --- a list of the max of each interval wyl --- a list of the rate of yellow lower violation wyu --- a list of the rate of yellow upper violation wrl --- a list of the rate of red lower violation wru --- a list of the rate of red upper violation wcnt --- a list of the total data counts wyl --- a list of the lower yellow limits wyu --- a list of the upper yellow limits wrl --- a list of the lower red limits wru --- a list of the upper red limits short_p: btime --- a list of time in sec from 1998.1.1 bdata --- a list of the mean of each interval bmed --- a list of the median of each interval bstd --- a list of the std of each interval bmin --- a list of the min of each interval bmax --- a list of the max of each interval byl --- a list of the rate of yellow lower violation byu --- a list of the rate of yellow upper violation brl --- a list of the rate of red lower violation bru --- a list of the rate of red upper violation bcnt --- a list of the total data counts byl --- a list of the lower yellow limits byu --- a list of the upper yellow limits brl --- a list of the lower red limits bru --- a list of the upper red limits long_p: --- all in one element list form ftime --- a mid time of the entier extracted data period fdata --- the mean of the entire extracted data fstd --- the std of the entire extracted data fmin --- the min of the entire extracte data fmax --- the max of the entire extracte data ylow --- the reate of yellow lower violation yupper --- the reate of yellow upper violation rlow --- the reate of red lower violation rupper --- the reate of red upper violation tcnt --- the total counts of the data ylow --- the lower yellow limit yup --- the upper yellow limit rlow --- the lower red limit rup --- the upper red limit """ # #--- week long data 5 min step # wtime = [] wdata = [] wmed = [] wstd = [] wmin = [] wmax = [] wyl = [] wyu = [] wrl = [] wru = [] wcnt = [] step2 = 86400 wstart = stop - 86400.0 * 14.0 #---- two weeks ago # #--- one year long data 1 hr step # btime = [] bdata = [] bmed = [] bstd = [] bmin = [] bmax = [] byl = [] byu = [] brl = [] bru = [] bcnt = [] wsave = [] vsave = [] # #--- extract data from archive # try: out = fetch.MSID(msid, start, stop) ok = ~out.bads tdata = out.vals[ok] ttime = out.times[ok] tind = [(tdata > -999) | (tdata <= -1000)] tdata = tdata[tind] ttime = ttime[tind] except: try: [ttime, tdata] = get_mta_fits_data(msid, start, stop) except: rdata = [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0], [0], [-9.0e9], [-9.0e9], [9.0e9], [9.0e9]] return [rdata, rdata, rdata] data = [] dtime = [] # #--- if the data is not given, the database desplay it as -999.999 (or similar); so drop them # for k in range(0, len(tdata)): try: test = int(float(tdata[k])) except: continue if int(abs(tdata[k])) == 999: continue data.append(tdata[k]) dtime.append(ttime[k]) data = numpy.array(data) dtime = numpy.array(dtime) # #--- get stat for the entire period # ftime = dtime.mean() fdata = data.mean() fmed = numpy.median(data) fstd = data.std() fmin = data.min() fmax = data.max() # #--- find the violation limits of that time # vlimits = find_violation_range(glim, ftime) # #--- get the violation rate of the entier period # [ylow, yupper, rlow, rupper, tcnt] = find_violation_rate(data, vlimits) # #--- if asked, devide the data into a smaller period (step size) # if step != 0: spos = 0 spos2 = 0 chk = 1 chk2 = 2 send = dtime[spos] + step send2 = dtime[spos2] + step2 for k in range(0, len(dtime)): if dtime[k] > wstart: if dtime[k] < send2: chk2 = 0 else: sdata = data[spos2:k] avg = sdata.mean() med = numpy.median(sdata) sig = sdata.std() amin = sdata.min() amax = sdata.max() ftime = dtime[spos2 + int(0.5 * (k - spos2))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(sdata, vlimits) wtime.append(ftime) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) spos2 = k send2 = dtime[k] + step2 chk2 = 1 else: send2 = dtime[spos2] + step2 if dtime[k] < send: chk = 0 else: rdata = data[spos:k] avg = rdata.mean() med = numpy.median(rdata) sig = rdata.std() amin = rdata.min() amax = rdata.max() ftime = dtime[spos + int(0.5 * (k - spos))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(rdata, vlimits) btime.append(ftime) bdata.append(avg) bmed.append(med) bstd.append(sig) bmin.append(amin) bmax.append(amax) byl.append(yl) byu.append(yu) brl.append(rl) bru.append(ru) bcnt.append(tot) vsave.append(vlimits) spos = k send = dtime[k] + step chk = 1 # #--- check whether there are any left over; if so add it to the data lists # if chk2 == 0: rdata = data[spos2:k] avg = rdata.mean() med = numpy.median(rdata) sig = rdata.std() amin = rdata.min() amax = rdata.max() ftime = dtime[spos2 + int(0.5 * (k - spos2))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(rdata, vlimits) wtime.append(dtime[spos2 + int(0.5 * (k - spos2))]) wdata.append(avg) wmed.append(med) wstd.append(sig) wmin.append(amin) wmax.append(amax) wyl.append(yl) wyu.append(yu) wrl.append(rl) wru.append(ru) wcnt.append(tot) wsave.append(vlimits) if chk == 0: rdata = data[spos:k] avg = rdata.mean() med = numpy.median(rdata) sig = rdata.std() amin = rdata.min() amax = rdata.max() ftime = dtime[spos + int(0.5 * (k - spos))] vlimits = find_violation_range(glim, ftime) [yl, yu, rl, ru, tot] = find_violation_rate(rdata, vlimits) btime.append(dtime[spos + int(0.5 * (k - spos))]) bdata.append(avg) bmed.append(med) bstd.append(sig) bmin.append(amin) bmax.append(amax) byl.append(yl) byu.append(yu) brl.append(rl) bru.append(ru) bcnt.append(tot) vsave.append(vlimits) week_p = [wtime, wdata, wmed, wstd, wmin, wmax, wyl, wyu, wrl, wru, wcnt] short_p = [btime, bdata, bmed, bstd, bmin, bmax, byl, byu, brl, bru, bcnt] # #--- adding limits to the table # vtemp = [[], [], [], []] for k in range(0, len(wsave)): for m in range(0, 4): vtemp[m].append(wsave[k][m]) week_p = week_p + vtemp # vtemp = [[], [], [], []] for k in range(0, len(vsave)): for m in range(0, 4): vtemp[m].append(vsave[k][m]) short_p = short_p + vtemp long_p = [[ftime], [fdata], [fmed], [fstd], [fmin], [fmax]] long_p = long_p + [[ylow], [yupper], [rlow], [rupper], [tcnt]] long_p = long_p + [[vlimits[0]], [vlimits[1]], [vlimits[2]], [vlimits[3]]] return [week_p, short_p, long_p]
def update_msid_data(msid_list='msid_list_fetch'): """ update all msid listed in msid_list input: msid_list --- a list of msids to processed. default: msid_list_fetch output: <msid>_data.fits/<msid>_short_data.fits """ start_time = time.strftime("%a, %d %b %Y %H:%M:%S", time.localtime()) # #--- create msid <---> unit dictionary # [udict, ddict] = ecf.read_unit_list() # #--- read mta database # mta_db = ecf.read_mta_database() # #--- read mta msid <---> sql msid conversion list # mta_cross = ecf.read_cross_check_table() # #--- read msid list # mfile = house_keeping + msid_list data = mcf.read_data_file(mfile) for ent in data: # #--- find msid and group name # mc = re.search('#', ent) if mc is not None: continue try: [msid, group] = re.split('\s+', ent) except: atemp = re.split('\s+', ent) msid = atemp[0] group = atemp[1] msid.strip() group.strip() # #--- get limit data table for the msid # try: tchk = convert_unit_indicator(udict[msid]) except: tchk = 0 ####glim = gsr.read_glimmon(msid, tchk) if msid in sp_limt_case_c: tchk = 1 ###print "I AM HERE TCHK : " + str(tchk) + "<--->" + str(udict[msid]) glim = get_limit(msid, tchk, mta_db, mta_cross) ###print "I AM HERE GLIM: " + str(glim) ###exit(1) # #--- update database # try: out = fetch.MSID(msid, '2017:001:00:00:00', '2017:002') print("MSID: " + msid) except: out = get_mta_fits_data(msid, '2017:001:00:00:00', '2017:002') if out == False: missed = house_keeping + '/missing_data' with open(missed, 'a') as fo fo.write(msid+ '\n') continue