def analyze_all(t,plot=False,pdffile=None): times, timestamp = find_solpnt(t) out = [] pp = None antlist = range(4) if plot: if pdffile: pp = PdfPages(pdffile) if len(times) > 1: for tstamp in times: t = util.Time(tstamp,format='lv') soldata = get_solpnt(t) proc = process_solpnt(soldata,antlist=antlist) out.append(fitall(proc,plot,pp))#=False)) else: t = util.Time(times[0],format='lv') soldata = get_solpnt(t) proc = process_solpnt(soldata,antlist=antlist) out.append(fitall(proc,plot,pp))#=False)) if plot: axtype = [' RA, DEC',' XEL, EL'] f, ax = plt.subplots(len(antlist), 2, sharex=True, sharey=True) f.set_size_inches(4,2*len(antlist),forward=True) t = util.Time(proc['Timestamp'],format='lv') f.suptitle('SOLPNT results for '+t.iso[:10],fontsize=18) for i in range(len(ax)): for j in range(len(ax[i])): ax[i][j].grid() ax[i][j].set_aspect('equal',adjustable='box-forced') ax[i][j].spines['right'].set_position('zero') ax[i][j].spines['top'].set_position('zero') ax[i][j].spines['left'].set_color('none') ax[i][j].spines['bottom'].set_color('none') ax[i][j].text(0.05,0.8,'Ant '+str(i+1)+axtype[j],transform=ax[i][j].transAxes,fontsize=10) for ant in antlist: for i in range(len(times)): if ant in out[i]['antlist']: iant = out[i]['antlist'].index(ant) x = [out[i]['HPol']['rao'][iant],out[i]['VPol']['rao'][iant]] y = [out[i]['HPol']['deco'][iant],out[i]['VPol']['deco'][iant]] ax[iant,0].plot(x,y,'.',x,y,'--') x = [out[i]['HPol']['xelo'][iant],out[i]['VPol']['xelo'][iant]] y = [out[i]['HPol']['elo'][iant],out[i]['VPol']['elo'][iant]] ax[iant,1].plot(x,y,'.',x,y,'--') if pp: pp.savefig(f) plt.close() pp.close() return out
def setSlot(self, event): st = util.Time(int(self.st_entry.get().split(":")[0]), int(self.st_entry.get().split(":")[1]), False) if self.en_entry.get() != "": en = util.Time(int(self.en_entry.get().split(":")[0]), int(self.en_entry.get().split(":")[1]), False) if (int(self.en_entry.get().split(":")[0]) < int( self.st_entry.get().split(":")[0])): en.hour += 12 if self.mode == "n": globals.schedule.week[self.dayIndex].blockedSlots.append( util.Slot(st, en, util.getTimeZone())) elif self.mode == "e": self.original_slot.start_time = st self.original_slot.end_time = en self.destroy()
def process_tsys(otp, proc, pol=None): ''' OTP contains tsys, of size [nant, npol, nf, nt], ut_mjd of size [nt], fghz of size [nf] and a 4-line ascii header. nant is 4 for prototype This only operates on one polarization at a time. If npol = 2, use pol=0 and pol=1 in two subsequent calls to do both polarizations. PROC contains mask of size [nant', nt', npnt], and an array of timestamps, to be compared with OTP['ut'] to synchronize them. nant' is the number of ants in the subarray. This assumes the first four ants are the prototype ants. ''' tsec1 = proc['tstamps'] # The otp times, if from miriad, are slightly off from exact times, so round to the ms tsec2 = (util.Time(otp['ut_mjd'], format='mjd').lv + 0.001).astype('int64').astype('float') idx1, idx2 = common_val_idx(tsec1, tsec2) if pol is None: nant, nf, nt = otp['tsys'].shape else: nant, npol, nf, nt = otp['tsys'].shape nant = len(proc['antlist']) # Make sure only small dishes are used antlist = proc['antlist'] # nf = len(np.where(otp['sfreq'] != 0)[0]) # Override nf with number of non-zero frequencies m = np.transpose( proc['mask'], (1, 0, 2))[0:nant, idx1, :] # mask now same number of ants and times as tsys print m.shape npt = len(m[0, 0, :]) if pol is None: tsys = otp['tsys'] else: tsys = otp['tsys'][:, pol, :, :] tsys = tsys[:, :, idx2] hpol = np.zeros([nf, npt, nant], 'float') pra = np.zeros([4, nf, nant], 'float') # Array of gaussian fits pdec = np.zeros([4, nf, nant], 'float') # Array of gaussian fits # Step through pointings, applying mask for ipnt in range(npt): for iant in range(nant): good = np.where(m[iant, :, ipnt])[0] if len(good) == 0: # This pointing location had no good tracking, so set values to Nan hpol[:, ipnt, iant] = np.nan else: # Set values to median of good-tracking period hpol[:, ipnt, iant] = np.median(tsys[antlist[iant], :, good], 0) # At this stage, hpol is of size [nf, npnt, nant] rao = hpol[:, :13, :] # First 13 pointings are RAO deco = hpol[:, 13:, :] # Second 13 pointings are DECO for iant in range(nant): for ifreq in range(nf): pra[:, ifreq, iant], x, y = gausfit(proc['rao'], rao[ifreq, :, iant]) pdec[:, ifreq, iant], x, y = gausfit(proc['deco'], deco[ifreq, :, iant]) return pra, pdec, rao, deco
def get_entries(time, filename, get_nearby): now_hr, now_minute = time.split(':') now = util.Time(int(now_hr), int(now_minute)) entries = list() with open(filename, 'r') as f: for line in f: route, startnr, hr, mn = [int(n) for n in line.split(' ')] time = util.Time(hr, mn) dt = util.diff_time(now, time) if (get_nearby and abs(dt) < 50) or (not get_nearby and dt > 0): entries.append((route, startnr, dt, time)) if get_nearby: entries.sort(key=lambda (r, s, dt, t): abs(dt)) else: entries.sort(key=lambda (r, s, dt, t): dt) num_entries = 50 return entries[:num_entries]
def is_slot_open(self): for i in globals.schedule.week[self.dayIndex].blockedSlots: t = util.Time(int(self.start_time.split(":")[0]), int(self.start_time.split(":")[1]), None) start_diff = i.start_time.get_diff(t, absolute_diff=False) end_diff = i.end_time.get_diff(t, absolute_diff=False) print(start_diff) print(end_diff) if ((start_diff.hour <= 0 or start_diff.minute < 0) and (end_diff.hour >= 0)): #or end_diff.minute >= 0 self.original_slot = i return False return True
def __init__( self, db_file=":memory:", db_log_path=None, task_db={}, load_minecraft_specs=True, load_block_types=True, ): self.db = sqlite3.connect(db_file) self.task_db = task_db self.banned_default_behaviors = [] # FIXME: move into triple store? self.other_players = {} self.pending_agent_placed_blocks = set() self.updateable_mems = [] self._safe_pickle_saved_attrs = {} if db_log_path: self._db_log_file = gzip.open(db_log_path + ".gz", "w") self._db_log_idx = 0 with open(SCHEMA, "r") as f: self._db_script(f.read()) self._load_schematics(load_minecraft_specs) self._load_block_types(load_block_types) # self.memids = {} # TODO: key is memid, value is sql table name or memory dict name self.all_tables = [ c[0] for c in self._db_read( "SELECT name FROM sqlite_master WHERE type='table';") ] self.dances = {} dance.add_default_dances(self) # create a "self" memory to reference in Triples self.self_memid = "0" * len(uuid.uuid4().hex) self._db_write("INSERT INTO Memories VALUES (?,?,?)", self.self_memid, "Memories", 0) self.tag(self.self_memid, "_agent") self.tag(self.self_memid, "_self") self.time = util.Time(mode="clock")
def dmp_tsys(t): ''' Given a start time as a Time() object t, find and dump the corresponding tsys values using Miriad (note that this only works on dpp or pipeline). Returns the filenames of the files created. ''' import socket, time import dump_tsys as dtsys hostname = socket.gethostname() if hostname != 'dpp' and hostname != 'pipeline': print 'Error: Cannot run dump_tsys() on host:',hostname return tnow = util.Time.now() t1 = t.lv - 10. # 10 s before given time t2 = t.lv + 300. # 300 s after given time trange = util.Time([t1,t2],format='lv') dtsys.dump_tsys(trange) # Check that the dump worked (wait 2 s for data) time.sleep(2) filestem ='t'+t.iso.replace('-','').replace(':','').replace(' ','')[:12]+'*.txt' xfiles = dtsys.glob.glob('/common/tmp/txt/x'+filestem) yfiles = dtsys.glob.glob('/common/tmp/txt/y'+filestem) return xfiles,yfiles
def rd_tsys(tsysfile,sfreqfile): ''' Read the Miriad log file output for a SOLPNT scan To generate the Miriad log file, enter the commands: varplt vis=<infile> xaxis=time yaxis=xtsys log=<tsys_file> varplt vis=<infile> xaxis=freq yaxis=wfreq log=<sfreq_file> where <infile> is the IDB or UDB file, and <tsys_file> is the Miriad log file name (.txt). Note that <sfreq_file> is a Miriad log file containing the starting frequencies, which is not yet implemented, but should be added to this routine. Major change to standardize output to ut_mdj, fghz, and order of indices of tsys as (nant, nf, ntimes) ''' with open(sfreqfile,'r') as s: sheader = [] for i in range(3): sheader.append(s.readline().strip()) try: nfrq = int(sheader[-1].split('freq(')[1].split(')')[0]) except: # This must be the format used by Jim McTiernan, so read another line sheader.append(s.readline().strip()) try: nfrq = int(sheader[-1].split('freq(')[1].split(')')[0]) except: print 'Could not successfully read sfreq file.' #return {} sfreq = np.zeros(nfrq,dtype='float') if (nfrq/4)*4 != nfrq: rlim = nfrq/4 + 1 else: rlim = nfrq/4 for i in range(rlim): line = s.readline() vals = line.split()[-4:] sfreq[i*4:min((i+1)*4,nfrq)] = vals nf = len(np.where(sfreq != 0)[0]) with open(tsysfile,'r') as f: header = [] for i in range(4): header.append(f.readline().strip()) # Set Time() object to value read from header line 2 (header[1]) dt = datetime.datetime.strptime(header[1].split(' ')[-1],'%y%b%d:%H:%M:%S.0') t = util.Time(dt,format='datetime') # Set base time in LabVIEW Timestamp format basetime = t.lv print 'Base Time is:',t.iso,'Timestamp:',basetime # Get number of columns from header line 4 (header[3]) sline = header[3].split(',') nant = int(sline[0].split('(')[1]) nfrq2 = int(sline[1].split(')')[0]) if nfrq != nfrq2: print tsysfile,'frequencies are not consistent with those in',sfreqfile return {} # Get number of times in file (just number of lines - 4 header lines, # divided by the number of frequencies, times nant/4 f.seek(0) nlinespertime = (nant-1)/4 + 1 nt = (sum(1 for line in f)-4)/nfrq2/nlinespertime print 'There are',nt,'times,',nfrq2,'frequencies, and',nant,'antennas in this file' # Get back to data start by rewind and skipping 4 header lines f.seek(0) for i in range(4): f.readline() out = np.zeros([nt,nfrq,nant],dtype='float') tstamp = np.zeros(nt,dtype='float') j = -1 while 1: try: line = f.readline() vals = line.strip().split() except: if line.strip() != '': print 'Error interpreting line:' print 'Line is:',line.strip()+';' break # This assumes Miriad dumps no more than 4 ants/line nvals = len(vals) if nvals == min(nant,4)+2: hms = vals[1].split(':') secs = int(vals[0])*86400 + int(hms[0])*3600 + int(hms[1])*60 + int(hms[2]) j += 1 tstamp[j] = basetime+secs vals = vals[2:] nvals -= 2 i = 0 ntot = 0 # Number inserted into "out" so far try: out[j,i,ntot:ntot+nvals] = np.array(vals,dtype='float') ntot += nvals if ntot >= nant: ntot = 0 i += 1 except: if line.strip() != '': print 'Error interpreting line at time,freq:',j,i print 'Line is:',line.strip()+';' break otp = {'ut_mjd':util.Time(tstamp[:j+1],format='lv').mjd,'fghz':sfreq[:nf],'tsys':np.swapaxes(out[:j+1,:nf,:],0,2),'header':header} return otp
def fitall(proc,plot=False,pp=None): ''' Fits SOLPNT data for all antennas contained in processed data proc, returned from process_solpnt() Prints results to terminal and optionally plots the gaussian fits. It also returns the fitted offsets. If pp is provided and is not None, the plots are written to a multipage PDF file. ''' t = util.Time(proc['Timestamp'],format='lv') nant = len(proc['antlist']) if plot: # row and column sharing f, ax = plt.subplots(7, 2, sharex='col', sharey='row') f.set_size_inches(10,14,forward=True) f.suptitle('Fits for SOLPNT scan at '+t.iso+' UT',fontsize=18) ax[0,0].set_title('RA Offset [blue=HPol, red=VPol]') ax[0,1].set_title('Dec Offset [blue=HPol, red=VPol]') # Temporary statement to make this work with no more than 7 antennas. if nant > 7: nant = 7 raoh = np.zeros(nant,dtype='float') decoh = np.zeros(nant,dtype='float') raov = np.zeros(nant,dtype='float') decov = np.zeros(nant,dtype='float') xeloh = np.zeros(nant,dtype='float') xelov = np.zeros(nant,dtype='float') eloh = np.zeros(nant,dtype='float') elov = np.zeros(nant,dtype='float') print 'SOLPNT solution for',t.iso print 'Ant XELO (deg) ELO (deg)' print ' HPol VPol Avg HPol VPol Avg' print '---- ------ ------ ------ ------ ------ ------' for i in range(nant): ant = proc['antlist'][i] [A, hrao, w, b], x, y = gausfit(proc['rao'],proc['hrao'][i,:]) if plot: ax[i,0].plot(x,y,proc['rao'],proc['hrao'][i,:],'o',label='Hpol') [A, vrao, w, b], x, y = gausfit(proc['rao'],proc['vrao'][i,:]) if plot: ax[i,0].plot(x,y,proc['rao'],proc['vrao'][i,:],'o',label='VPol') ax[i,0].text(0.05,0.8,'Ant '+str(ant+1),transform=ax[i,0].transAxes,fontsize=14) ax[i,0].text(0.05,0.65,'H = {:6.3f}'.format(hrao/10000.),transform=ax[i,0].transAxes) ax[i,0].text(0.05,0.5,'V = {:6.3f}'.format(vrao/10000.),transform=ax[i,0].transAxes) [A, hdeco, w, b], x, y = gausfit(proc['deco'],proc['hdeco'][i,:]) if plot: ax[i,1].plot(x,y,proc['deco'],proc['hdeco'][i,:],'o',label='HPol') [A, vdeco, w, b], x, y = gausfit(proc['deco'],proc['vdeco'][i,:]) if plot: ax[i,1].plot(x,y,proc['deco'],proc['vdeco'][i,:],'o',label='VPol') ax[i,1].text(0.05,0.8,'Ant '+str(ant+1),transform=ax[i,1].transAxes,fontsize=14) ax[i,1].text(0.05,0.65,'H = {:6.3f}'.format(hdeco/10000.),transform=ax[i,1].transAxes) ax[i,1].text(0.05,0.5,'V = {:6.3f}'.format(vdeco/10000.),transform=ax[i,1].transAxes) if sys.platform[:5] == 'linux': # Convert rao, deco to azo, elo (returned in radians) cosdec = np.cos(proc['dec0']) hxelo, helo = dradec2dazel(proc['ra0'],proc['dec0'], t,hrao*np.pi/10000./180./cosdec,hdeco*np.pi/10000./180.) vxelo, velo = dradec2dazel(proc['ra0'],proc['dec0'], t,vrao*np.pi/10000./180./cosdec,vdeco*np.pi/10000./180.) else: hxelo, helo = hrao*np.pi/10000./180., hdeco*np.pi/10000./180. vxelo, velo = vrao*np.pi/10000./180., vdeco*np.pi/10000./180. # Print table of XEL and EL offsets as degrees print ' {:2d} {:6.3f} {:6.3f} {:6.3f} {:6.3f} {:6.3f} {:6.3f}'.format( proc['antlist'][i]+1, hxelo*180./np.pi, vxelo*180./np.pi,(hxelo + vxelo)*180./np.pi/2., helo*180./np.pi,velo*180./np.pi,(helo+velo)*180./np.pi/2.) if plot: ax[i,0].text(0.65,0.65,'Hxel = {:6.3f}'.format(hxelo*180./np.pi),transform=ax[i,0].transAxes) ax[i,0].text(0.65,0.5,'Vxel = {:6.3f}'.format(vxelo*180./np.pi),transform=ax[i,0].transAxes) ax[i,1].text(0.65,0.65,'Hel = {:6.3f}'.format(helo*180./np.pi),transform=ax[i,1].transAxes) ax[i,1].text(0.65,0.5,'Vel = {:6.3f}'.format(velo*180./np.pi),transform=ax[i,1].transAxes) raoh[i] = hrao/10000. # degrees raov[i] = vrao/10000. # degrees decoh[i] = hdeco/10000. # degrees decov[i] = vdeco/10000. # degrees xeloh[i] = hxelo*180./np.pi # degrees xelov[i] = vxelo*180./np.pi # degrees eloh[i] = helo*180./np.pi # degrees elov[i] = velo*180./np.pi # degrees if plot: ax[nant-1,0].set_xlabel('RA Offset [0.0001 deg]') ax[nant-1,1].set_xlabel('Dec Offset [0.0001 deg]') if pp: pp.savefig(f) plt.close() else: plt.show() return {'Timestamp':proc['Timestamp'],'antlist':proc['antlist'], 'HPol':{'rao':raoh, 'deco':decoh, 'elo':eloh, 'xelo':xeloh}, 'VPol':{'rao':raov, 'deco':decov, 'elo':elov, 'xelo':xelov}}
def get_solpnt(t=None): ''' Get the SOLPNT data from the SQL database, occurring after time given in the Time() object t. If omitted, the first SOLPNT scan for the current day is used (if it exists). ''' import dbutil tstamps, timestamp = find_solpnt(t) # Find first SOLPNTCAL occurring after timestamp (time given by Time() object) if tstamps != []: print 'SOLPNTCAL scans were found at ', for tstamp in tstamps: if type(tstamp) is np.ndarray: # Annoyingly necessary when only one time in tstamps tstamp = tstamp[0] t1 = util.Time(tstamp,format='lv') print t1.iso,';', print ' ' good = np.where(tstamps >= timestamp)[0] # This is the timestamp of the first SOLPNTCAL scan after given time if good.shape[0] == 0: stimestamp = tstamps[0] else: stimestamp = tstamps[good][0] else: print 'Warning: No SOLPNTCAL scan found, so interpreting given time as SOLPNTCAL time.' stimestamp = timestamp # Grab 300 records after the start time cursor = dbutil.get_cursor() # Now version independent! verstr = dbutil.find_table_version(cursor,timestamp) if verstr is None: print 'No stateframe table found for the given time.' return {} solpntdict = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=15,timestamp=stimestamp,nrecs=300) # Need dimension-1 data to get antennas in subarray -- Note: sometimes the antenna list # is zero (an unlikely value!) around the time of the start of a scan, so keep searching # first 100 records until non-zero: for i in range(100): blah = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=1,timestamp=stimestamp+i,nrecs=1) if blah['LODM_Subarray1'][0] != 0: break cursor.close() sub1 = blah['LODM_Subarray1'][0] subarray1 = [] antlist = [] for i in range(16): subarray1.append(sub1 & (1<<i) > 0) if subarray1[-1]: antlist.append(i) # print 'Antlist:',antlist # rao = np.zeros([15,300],dtype='int') # deco = np.zeros([15,300],dtype='int') # trk = np.zeros([15,300],dtype='bool') # hpol = np.zeros([15,300],dtype='float') # vpol = np.zeros([15,300],dtype='float') # ra = np.zeros(15,dtype='float') # dec = np.zeros(15,dtype='float') ra = (solpntdict['Ante_Cont_RAVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float') dec = (solpntdict['Ante_Cont_DecVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float') hpol = (solpntdict['Ante_Fron_FEM_HPol_Voltage'][:,antlist]).astype('float') vpol = (solpntdict['Ante_Fron_FEM_VPol_Voltage'][:,antlist]).astype('float') rao = (solpntdict['Ante_Cont_RAOffset'][:,antlist]).astype('float') deco = (solpntdict['Ante_Cont_DecOffset'][:,antlist]).astype('float') times = solpntdict['Timestamp'][:,0].astype('int64').astype('float') # Convert pointing info to track information outdict = stateframe.azel_from_sqldict(solpntdict) trk = np.logical_and(outdict['dAzimuth'][:,antlist]<0.0020,outdict['dElevation'][:,antlist]<0.0020) return {'Timestamp':stimestamp,'tstamps':times,'antlist':antlist,'trjfile':'SOLPNT.TRJ','ra':ra,'dec':dec, 'rao':rao,'deco':deco,'trk':trk,'hpol':hpol,'vpol':vpol}
def transition_func(state, action): # Update location and time travel_stats = travel_data.stat_travel_time( travel_data.to_pairs_list( travel_data.get_travel_data(), travel_data.get_locations())) curr_location = state.location next_location = action.location travel_time = travel_stats[(curr_location, next_location)]["mean"] time_of_day = state.time_of_day + travel_time # New request to add: new_request_list = state.request_history + [Request(location=next_location, time_of_day=time_of_day, day_of_week=state.day_of_week)] # Dynamics of people moving new_present_map = state.person_present_map for person in new_present_map: schedule = schedule_data.get_schedule_data()[person] if new_present_map[person] is None or not new_present_map[person]: # Check if person will arrive if "schedule" in schedule: # Room with fixed schedules intervals = util.parse_schedule(schedule["schedule"]) # Each person's arrival to their office is modeled as a normal distribution centered around their scheduled time of arrival. p_intervals = (probability_interval(interval, state.time_of_day, time_of_day, arrival_stddev, True) for interval in intervals) for idx, prob in enumerate(p_intervals): if random.random() < prob: new_present_map[person] = int(sample_probability_interval(intervals[idx], state.time_of_day, time_of_day, arrival_stddev, True)) break elif "intervals-daily" in schedule: # Open area daily_intervals = schedule["intervals-daily"] p_second_arrival = daily_intervals / 24.0 / 60.0 / 60.0 p_arrival = p_second_arrival * travel_time if random.random() < p_arrival: new_present_map[person] = time_of_day else: # Remove person if appropriate # Scheduled offices if "schedule" in schedule: intervals = util.parse_schedule(schedule["schedule"]) # Each person's departure from their office is modeled as a normal distribution centered around their scheduled time of departure. p_intervals = (probability_interval(interval, state.time_of_day, time_of_day, departure_stddev, False) for interval in intervals) for idx, prob in enumerate(p_intervals): if random.random() < prob: new_present_map[person] = None break # Non-scheduled offices (mostly open spaces) elif "intervals-daily" in schedule: interval = util.Interval( util.Time(state.time_of_day, new_present_map[person]), util.Time(state.time_of_day, new_present_map[person] + schedule["duration"])) p_interval = probability_interval(interval, state.time_of_day, time_of_day, open_stddev, False) if random.random() < p_interval: new_present_map[person] = None # Resulting state return FullState(location=next_location, time_of_day=time_of_day, day_of_week=state.day_of_week, person_present_map=new_present_map, request_history=new_request_list)
# days_in_week = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri'] hours_in_day = [ '0800', '0815', '0830', '0845', '0900', '0915', '0930', '0945', '1000', '1015', '1030', '1045', '1100', '1115', '1130', '1145', '1200', '1215', '1230', '1245', '1300', '1315', '1330', '1345', '1400', '1415', '1430', '1445', '1500', '1515', '1530', '1545', '1600' ] # Generate domain for a time variables. time_seg_idx = [] # it's the scheduling horizon, maybe call it that. for i in policy_data.calendar_work_days: for j in hours_in_day: time = util.Time(i, j, len(time_seg_idx)) time_seg_idx.append(time) # ----------------------------------------------------------------------------- # Optimization variables # ----------------------------------------------------------------------------- # the main loop of making the variables. mdl = CpoModel() all_visits = [] # Set admittance variables. for pat in patient_list: name = 'z_' + str(pat.id) pat.is_admitted_var = mdl.binary_var(name=name)
def get_timer(self): timer = util.Time() return timer
self.startnr = startnr self.hour = hour self.minute = minute self.kind = int_to_kind(kind) data = list() for fn in sys.argv[3:5]: with open(fn, 'r') as f: for line in f: routenr, startnr, hr, minute, kind = [int(n) for n in line.split()] data.append(BusInfo(routenr, startnr, hr, minute, kind)) now_h, now_m = [int(n) for n in sys.argv[2].split(':')] now = util.Time(now_h, now_m) passed = set([(bi.routenr, bi.startnr) for bi in data if bi.kind == Kind.Passed]) have_gps_data = set([(bi.routenr, bi.startnr) for bi in data if bi.kind == Kind.GPS]) data = [bi for bi in data if (bi.routenr, bi.startnr) not in passed] data = [ bi for bi in data if (bi.routenr, bi.startnr) not in have_gps_data or bi.kind == Kind.GPS ] data.sort(key=lambda bi: util.diff_time(now, util.Time(bi.hour, bi.minute))) print sys.argv[1] print sys.argv[2] for bi in data[:7]:
def process_tsys(otp, proc, pol=None, skycal=None): ''' OTP contains tsys, of size [nant, npol, nf, nt], ut_mjd of size [nt], fghz of size [nf] and a 4-line ascii header. nant is 4 for prototype This only operates on one polarization at a time. If npol = 2, use pol=0 and pol=1 in two subsequent calls to do both polarizations. PROC contains mask of size [nant', nt', npnt], and an array of timestamps, to be compared with OTP['ut'] to synchronize them. nant' is the number of ants in the subarray. This assumes the first four ants are the prototype ants. skycal is an optional dictionary of SKYCALTEST results. If provided (not None), the data in the 'offsun' key will be used to create pseudo-10-degree pointing offset data for the fitting. This only works if SKYCALTEST and SOLPNTCAL scans use the same frequencies and antennas, which should always be the case. ''' from disk_conv import disk_conv fghz, a, aout = disk_conv(otp['fghz']) aout *= 10000. / (2 * np.sqrt(np.log(2))) tsec1 = proc['tstamps'] # The otp times, if from miriad, are slightly off from exact times, so round to the ms tsec2 = (util.Time(otp['ut_mjd'], format='mjd').lv + 0.001).astype('int64').astype('float') idx1, idx2 = util.common_val_idx(tsec1, tsec2) if pol is None: nant, nf, nt = otp['tsys'].shape else: nant, npol, nf, nt = otp['tsys'].shape nant = len(proc['antlist']) # Make sure only small dishes are used antlist = proc['antlist'] # nf = len(np.where(otp['sfreq'] != 0)[0]) # Override nf with number of non-zero frequencies m = np.transpose( proc['mask'], (1, 0, 2))[0:nant, idx1, :] # mask now same number of ants and times as tsys print m.shape npt = len(m[0, 0, :]) if pol is None: tsys = otp['tsys'] else: tsys = otp['tsys'][:, pol, :, :] tsys = tsys[:, :, idx2] hpol = np.zeros([nf, npt, nant], 'float') pra = np.zeros([4, nf, nant], 'float') # Array of gaussian fits pdec = np.zeros([4, nf, nant], 'float') # Array of gaussian fits # Step through pointings, applying mask for ipnt in range(npt): for iant in range(nant): good = np.where(m[iant, :, ipnt])[0] if len(good) == 0: # This pointing location had no good tracking, so set values to Nan hpol[:, ipnt, iant] = np.nan else: # Set values to median of good-tracking period hpol[:, ipnt, iant] = np.median(tsys[antlist[iant], :, good], 0) raopts = proc['rao'] decopts = proc['deco'] # At this stage, hpol is of size [nf, npnt, nant] rao = hpol[:, :13, :] # First 13 pointings are RAO deco = hpol[:, 13:, :] # Second 13 pointings are DECO # Handle skycal offsun modification, provided the number of frequencies # and number of antennas in skycal and solpnt match (they should!). Otherwise, skip it. if (not pol is None) and skycal: skynant, skynpol, skynf = skycal['offsun'].shape if nf == skynf and nant == skynant: # Insert 10-degree offsets before and after actual offsets raopts = np.array([-100000.] + proc['rao'].tolist() + [100000.]) decopts = np.array([-100000.] + proc['deco'].tolist() + [100000.]) faroff = np.swapaxes( skycal['offsun'][:, pol, :] - skycal['rcvr_bgd'][:, pol, :], 0, 1) faroff.shape = (nf, 1, nant) # The faroff values are only needed for frequencies below about 2.75 GHz, # so set them to NaN above that frequency (gaussfit will then ignore them) bad, = np.where(otp['fghz'] > 2.75) faroff[bad] = np.nan # Insert faroff values corresponding to 10-degree offsets rao = np.concatenate((faroff, hpol[:, :13, :], faroff), 1) deco = np.concatenate((faroff, hpol[:, 13:, :], faroff), 1) for iant in range(nant): for ifreq in range(nf): ydata = rao[ifreq, :, iant] allzero = len(ydata.nonzero()[0]) == 0 # True if all data are zero xdata = raopts ymax = np.nanmax(ydata) ymin = np.nanmin(ydata) yrange = ymax - ymin low_bounds = [ 0.1 * yrange, -30000., aout[ifreq] * 0.9, ymin - 0.1 * yrange ] high_bounds = [ 1.0 * yrange, 30000., aout[ifreq] * 1.1, ymin + 0.1 * yrange ] if allzero: high_bounds[ 0] = 1 # Ensure that high bounds are greater than low bounds high_bounds[3] = 1 bounds = (low_bounds, high_bounds) #if iant == 1 and ifreq == 30: # import pdb; pdb.set_trace() pra[:, ifreq, iant], x, y = gausfit(xdata, ydata, bounds=bounds) ydata = deco[ifreq, :, iant] allzero = len(ydata.nonzero()[0]) == 0 # True if all data are zero xdata = decopts ymax = np.nanmax(ydata) ymin = np.nanmin(ydata) yrange = ymax - ymin low_bounds = [ 0.1 * yrange, -30000., aout[ifreq] * 0.9, ymin - 0.1 * yrange ] high_bounds = [ 1.0 * yrange, 30000., aout[ifreq] * 1.1, ymin + 0.1 * yrange ] # low_bounds = [0.1*np.nanmax(ydata), -30000., aout[ifreq]*0.9, np.nanmin(ydata)-0.1*np.nanmax(ydata)] # high_bounds = [1.0*np.nanmax(ydata), 30000., aout[ifreq]*1.1, np.nanmin(ydata)+0.1*np.nanmax(ydata)] if allzero: high_bounds[ 0] = 1 # Ensure that high bounds are greater than low bounds high_bounds[3] = 1 bounds = (low_bounds, high_bounds) pdec[:, ifreq, iant], x, y = gausfit(xdata, ydata, bounds=bounds) return pra, pdec, rao, deco