def tab1_update_addStrID(): global dftmp, tab1_tim_square, timestart, database_dir, tab1_selected_SRC_dspec_squarez if tab1_selected_SRC_dspec_square: time0, time1 = dftmp['time'].min() + timestart, dftmp['time'].max() + timestart freq0, freq1 = dftmp['freq'].min(), dftmp['freq'].max() date_char = jdutil.jd_to_datetime(timestart / 3600. / 24.) date_char = date_char.strftime('%Y-%b-%d') t0_char = jdutil.jd_to_datetime(time0 / 3600. / 24.) t0_char = t0_char.strftime('%H:%M:%S') + '.{:03d}'.format(int(round(t0_char.microsecond / 1e3))) t1_char = jdutil.jd_to_datetime(time1 / 3600. / 24.) t1_char = t1_char.strftime('%H:%M:%S') + '.{:03d}'.format(int(round(t1_char.microsecond / 1e3))) time0, time1 = (time0 / 86400. - 2400000.5) * 24. * 3600., (time1 / 86400. - 2400000.5) * 24. * 3600. StrID = pd.DataFrame(dict(time=[[time0, time1, time1, time0]], freq=[[freq0, freq0, freq1, freq1]], str_id=[[tab1_input_StrID.value]], date=[[date_char]], timeran=[[t0_char + '~' + t1_char]], freqran=[["{:.3f}~{:.3f} GHz".format(freq0, freq1)]])) StrIDList = pd.read_json(database_dir + event_id + 'StrID_list_tmp.json') StrIDList = pd.concat([StrIDList, StrID]) StrIDList = StrIDList.sort_values(by='timeran', ascending=1) StrIDList.index = range(StrIDList.index.size) StrIDList.to_json(database_dir + event_id + 'StrID_list_tmp.json') StrIDList['time'] = [ll - tab1_tim_square[0] for ll in StrIDList['time']] tab1_render_patch.data_source.data = ColumnDataSource(StrIDList).data tab1_Div_Tb.text = """<p>added <b>""" + tab1_input_StrID.value + """</b> to the list</p>""" else: tab1_Div_Tb.text = """<p><b>Warning: No time & freq range selected.
def BckGrdDataGrab(hdf): dat=tables.openFile(hdf) zen = dat.root.SplineMPEMod.col('zenith') azi = dat.root.SplineMPEMod.col('azimuth') err = dat.root.ReScaled_Paraboloid_Sigma_SplineMPEMod.col('value') nch = dat.root.FinalLevelNch.col('value') run = dat.root.I3EventHeader.col('Run') time_mjd = dat.root.timeMJD.col('value') time_jd = time_mjd + 2400000.5 time=[] for temp in time_jd: time.append(jdutil.jd_to_datetime(temp)) time = numpy.array(time) ontime_bool = numpy.ones(len(nch)) == 1. for indy,event_time in enumerate(time_mjd): flip_bool=False jdevent=event_time+astrodate.MJD_0 for burst in grbs.grbs: jdburst_start = astrodate.JulianDate(burst.t_start) jdburst_end = astrodate.JulianDate(burst.t_end) cleared = False if jdevent - jdburst_start.jd < -0.0833: cleared=True if jdevent - jdburst_end.jd > 0.0833: cleared=True if not cleared: flip_bool = True if flip_bool: ontime_bool[indy] = False data_vars = Vars() data_vars.zenith = zen[ontime_bool] data_vars.azimuth = azi[ontime_bool] data_vars.error = err[ontime_bool] data_vars.nchan = nch[ontime_bool] data_vars.timeMJD = time_mjd[ontime_bool] data_vars.run = run[ontime_bool] data_vars.time = time[ontime_bool] data_vars.eproxy = 5.0*data_vars.nchan return data_vars
def DataGrab(hdf): dat=tables.openFile(hdf) zen = dat.root.SplineMPEMod.col('zenith') azi = dat.root.SplineMPEMod.col('azimuth') err = dat.root.ReScaled_Paraboloid_Sigma_SplineMPEMod.col('value') nch = dat.root.FinalLevelNch.col('value') run = dat.root.I3EventHeader.col('Run') time_mjd = dat.root.timeMJD.col('value') time_jd = time_mjd + 2400000.5 time=[] for temp in time_jd: time.append(jdutil.jd_to_datetime(temp)) time = numpy.array(time) data_vars = Vars() data_vars.zenith = zen data_vars.azimuth = azi data_vars.error = err data_vars.nchan = nch data_vars.run = run data_vars.timeMJD = time_mjd data_vars.time = time data_vars.eproxy = 5.0*data_vars.nchan return data_vars
def Add_gattrs(ff): lon_min = min(data['LON']) lon_max = max(data['LON']) lat_min = min(data['LAT']) lat_max = max(data['LAT']) start_time = min(data.data['Julian']) end_time = max(data.data['Julian']) dur_time = (end_time - start_time) * 24.0 * 3600.0 start_time = jdutil.jd_to_datetime(start_time) start_time_s = "%s-%02d-%02dT%02d:%02d:%02dZ" % ( start_time.year, start_time.month, start_time.day, start_time.hour, start_time.minute, start_time.second) end_time = jdutil.jd_to_datetime(end_time) end_time_s = "%s-%02d-%02dT%02d:%02d:%02dZ" % ( end_time.year, end_time.month, end_time.day, end_time.hour, end_time.minute, end_time.second) version = out_file.split('_')[1] #start_time_s = time.strftime(time_fmt,time.gmtime(float(start_time))) #end_time_s = time.strftime(time_fmt,time.gmtime(float(end_time))) ff.ncei_template_version = "NCEI_NetCDF_Point_Template_v2.0" ff.featureType = "point" ff.title = "International Comprehensive Ocean-Atmosphere Data Set (ICOADS) %s data collected from %s to %s." % ( version, start_time_s, end_time_s) ff.summary = "This file contains ICOADS %s data in netCDF4 format collected from %s to %s. The International Comprehensive Ocean-Atmosphere Data Set (ICOADS) offers surface marine data spanning the past three centuries, and simple gridded monthly summary products for 2-degree latitude x 2-degree longitude boxes back to 1800 (and 1degreex1degree boxes since 1960)--these data and products are freely distributed worldwide. As it contains observations from many different observing systems encompassing the evolution of measurement technology over hundreds of years, ICOADS is probably the most complete and heterogeneous collection of surface marine data in existence." % ( version, start_time_s, end_time_s) ff.keywords = get_keywords(data) ff.Conventions = "CF-1.6, ACDD-1.3" ff.id = out_file.split('.nc')[0].replace('IMMA1', 'ICOADS') ff.naming_authority = "gov.noaa.ncei" #ff.source = "http://rda.ucar.edu/data/ds548.0/imma1_r3.0.0/%s.tar" %out_file.split('-')[0] ff.source = "%s.gz" % out_file.split('.nc')[0] ff.processing_level = "Restructured from IMMA1 format to NetCDF4 format." ff.acknowledgement = "Conversion of ICOADS data from IMMA1 to netCDF format by NCEI is supported by the NOAA Big Earth Data Initiative (BEDI)." ff.license = "These data may be redistributed and used without restriction." ff.standard_name_vocabulary = "CF Standard Name Table v31" ff.date_created = time.strftime(time_fmt, time.gmtime()) ff.creator_name = "NCEI" ff.creator_email = "*****@*****.**" ff.creator_url = "https://www.ncei.noaa.gov/" ff.institution = "National Centers for Environmental Information (NCEI), NOAA" ff.project = "International Comprehensive Ocean-Atmosphere Data Set (ICOADS) Project" ff.publisher_name = "NCEI" ff.publisher_email = "*****@*****.**" ff.publisher_url = "https://www.ncei.noaa.gov/" ff.geospatial_bounds = "POLYGON ((%.4f %.4f,%.4f %.4f,%.4f %.4f,%.4f %.4f,%.4f %.4f))" % ( lon_min, lat_min, lon_min, lat_max, lon_max, lat_max, lon_max, lat_min, lon_min, lat_min) ff.geospatial_bounds_crs = "EPSG:4326" ff.geospatial_lat_min = float("%.4f" % (lat_min)) ff.geospatial_lat_max = float("%.4f" % (lat_max)) ff.geospatial_lon_min = float("%.4f" % (lon_min)) ff.geospatial_lon_max = float("%.4f" % (lon_max)) ff.geospatial_lat_units = "degrees_north" ff.geospatial_lon_units = "degrees_east" ff.time_coverage_start = start_time_s ff.time_coverage_end = end_time_s ff.time_coverage_duration = 'P' + duration(dur_time) ff.time_coverage_resolution = "vary" ff.uuid = str(uuid.uuid4()) ff.sea_name = "World-Wide Distribution" ff.creator_type = "group" ff.creator_institution = "NOAA National Centers for Environmental Information (NCEI)" ff.publisher_type = "institution" ff.publisher_institution = "NOAA National Centers for Environmental Information (NCEI)" ff.program = "" ff.contributor_name = "Zhankun Wang; ICOADS team" ff.contributor_role = "ICOADS Data Conversion to NetCDF; ICOADS IMMA1 Data Provider" ff.date_modified = time.strftime(time_fmt, time.gmtime()) ff.date_issued = time.strftime(time_fmt, time.gmtime()) ff.date_metadata_modified = time.strftime(time_fmt, time.gmtime()) ff.product_version = "ICOADS %s netCDF4" % version ff.keywords_vocabulary = "Global Change Master Directory (GCMD) 2015. GCMD Keywords, Version 8.1." ff.cdm_data_type = 'Point' #ff.metadata_link = 'http://rda.ucar.edu/datasets/ds548.0/#!docs' ff.metadata_link = '' if len(set(data.data['IM'])) == 1: ff.IMMA_Version = str(data.data['IM'][0]) else: print('%s: check IMMA version' % out_file) if len(set(data.data['RN1'])) == 1: ff.Release_Number_Primary = str(data.data['RN1'][0]) else: print('%s: check Release_Number_Primary' % out_file) if len(set(data.data['RN2'])) == 1: ff.Release_Number_Secondary = str(data.data['RN2'][0]) else: print('%s: check Release_Number_Secondary' % out_file) if len(set(data.data['RN3'])) == 1: ff.Release_Number_Tertiary = str(data.data['RN3'][0]) else: print('%s: check Release_Number_Tertiary' % out_file) if len(set(data.data['RSA'])) == 1: ff.Release_status_indicator = str(data.data['RSA'][0]) else: print('%s: check RSA' % out_file) #ff.comment = "" ff.references = 'http://rda.ucar.edu/datasets/ds548.0/docs/R3.0-citation.pdf' ff.history = time.strftime(time_fmt, time.gmtime( )) + ": Converted from IMMA1 format to netCDF4 format by Z.W. "
def plt_dspec_old(mspath=None, specfile=None, pol='I', dmin=None, dmax=None, fig=None): # Set up variables if pol != 'RR' and pol != 'LL' and pol != 'RRLL' and pol != 'I' and pol != 'V' and pol != 'IV': print "Please enter 'RR', 'LL', 'RRLL', 'I', 'V', 'IV' for pol" return 0 if not mspath: mspath = '.' specdata = np.load(mspath + '/' + specfile) spec = specdata['spec'] tim = specdata['tim'] freq = specdata['freq'] # setup plot parameters print 'ploting dynamic spectrum...' # mask the channels from 512 to 519 (no observation) spec = np.ma.array(spec) spec[:, 512:519, :] = np.ma.masked spec_med = np.median(np.absolute(spec)) # set the time axis ntim = tim.shape if ntim[0] < 20: xticks = np.arange((ntim[0] - 1) / 2 + 1) * 2 elif ntim[0] < 100: xticks = np.arange((ntim[0] - 1) / 20 + 1) * 20 elif ntim[0] < 600: xticks = np.arange((ntim[0] - 1) / 100 + 1) * 100 elif ntim[0] < 2400: xticks = np.arange((ntim[0] - 1) / 400 + 1) * 400 elif ntim[0] < 12000: # 1 min per step tstart = np.fix(tim[0] / 60.) + 1 xstart = np.abs(tim - tstart * 60.).argmin() xticks = np.arange(ntim[0] / 1200) * 1200 + xstart elif ntim[0] > 12000: xticks = np.arange(ntim[0] / 6000 + 1) * 6000 nticks = xticks.shape xticktims = [] for i in range(nticks[0]): # xticktim0=qa.time(qa.quantity(tim[xticks[i]],'s')) tims = tim[xticks[i]] # in seconds tims_jd = jdutil.mjd_to_jd(tims / 3600. / 24.) # to julian date tims_dt = jdutil.jd_to_datetime(tims_jd) tims_dt2 = tims_dt + datetime.timedelta( seconds=round(tims_dt.microsecond / 1e6)) tims_char = tims_dt2.strftime('%H:%M:%S') xticktims.append(tims_char) xticks = list(xticks) # do the plot f = plt.figure(figsize=(10, 6), dpi=100) if not dmin: dmin = spec_med / 20. if not dmax: dmax = spec_med * 5. if pol != 'RRLL' and pol != 'IV': ax = f.add_subplot(111) if pol == 'RR': spec_plt = np.absolute(spec[0, :, :]) elif pol == 'LL': spec_plt = np.absolute(spec[1, :, :]) elif pol == 'I': spec_plt = (np.absolute(spec[0, :, :]) + np.absolute(spec[1, :, :])) / 2. elif pol == 'V': spec_plt = (np.absolute(spec[0, :, :]) - np.absolute(spec[1, :, :])) / 2. # ax.imshow(spec_plt,aspect='auto',origin='lower',extent=[0,ntim[0]-1,np.min(freq)/1e6,np.max(freq)/1e6],\ # vmin=dmin,vmax=dmax,interpolation='none') # ax.set_xticks(xticks) # ax.set_xticklabels(xticktims) tim_ = tim - tim[0] freqghz = freq / 1e9 ax.pcolormesh(tim_, freqghz, spec_plt, cmap='jet', vmin=dmin, vmax=dmax) ax.set_xlim([tim_[0], tim_[-1]]) ax.set_ylim([freqghz[0], freqghz[-1]]) ax.set_xlabel('Time (pixel)') ax.set_ylabel('Frequency (GHz)') ax.set_title('VLA dynamic spectrum for pol ' + pol) ax.set_autoscale_on(False) else: R_plot = np.absolute(spec[0, :, :]) L_plot = np.absolute(spec[1, :, :]) I_plot = (Rplot + Lplot) / 2. V_plot = (Rplot - Lplot) / 2. if pol == 'RRLL': spec_plt_1 = R_plot spec_plt_2 = L_plot if pol == 'IV': spec_plt_1 = I_plot spec_plt_2 = V_plot ax1 = f.add_subplot(211) ax1.imshow(spec_plt_1, aspect='auto', origin='lower', extent=[0, ntim[0] - 1, np.min(freq) / 1e6, np.max(freq) / 1e6], \ vmin=dmin, vmax=dmax, interpolation='none') ax1.set_xticks(xticks) ax1.set_xticklabels(xticktims) ax1.set_xlabel('Universal Time (50 ms/pixel)') ax1.set_ylabel('Frequency (MHz)') ax2.set_title('VLA dynm spec for pol ' + pol[0]) ax1.set_autoscale_on(False) ax2 = f.add_subplot(212) ax2.imshow(spec_plt_2, aspect='auto', origin='lower', extent=[0, ntim[0] - 1, np.min(freq) / 1e6, np.max(freq) / 1e6], \ vmin=dmin, vmax=dmax, interpolation='none') ax2.set_xticks(xticks) ax2.set_xticklabels(xticktims) ax2.set_xlabel('Universal Time (50 ms/pixel)') ax2.set_ylabel('Frequency (MHz)') ax2.set_title('VLA dynm spec for pol ' + pol[-1]) ax2.set_autoscale_on(False) if fig: figfile = specfile[:(specfile.find('spec'))] + pol + '.pdf' f.savefig(mspath + '/' + figfile) # plt.close() return (f, ax)
freq=[[freq[0], freq[0], freq[-1], freq[-1]]], str_id=[['U01']]) with open('structure_id_list', 'wb') as f: pickle.dump(StructureIdList, f) # pb = progressbar(idx_tim.size, "*") # for ii in range(8405,len(idx_selec[0])): # for ii in range(0,idx_tim.size): timestrs = [] for ii in range(len(xx)): t_int = 0.05 f_int = 2.0 # MHz t0 = xx[ii] # -0.5*t_int timestr0 = jdutil.jd_to_datetime(t0 / 3600. / 24.) timestr = timestr0.strftime('%Y-%m-%dT%H%M%S') + '.{:03d}'.format( int(round(timestr0.microsecond / 1e3))) timestrs.append( timestr0.strftime('%H:%M:%S') + '.{:03d}'.format(int(round(timestr0.microsecond / 1e3)))) f0 = yy[ii] * 1e3 freqstr = '{:d}MHz'.format(int(round(f0))) fits_local.append(structure_id + '_' + timestr + '_' + freqstr + '.slfcal.image.cutout.fits') fits_global.append(structure_id + '_' + timestr + '_' + freqstr + '.slfcal.image.fits') Gauss_params = [] start_timestamp = time.time()
elif tab1_pol == 'V': tab1_spec_plt = (tab1_spec[0, bl_index, :, :] - tab1_spec[1, bl_index, :, :]) / 2. tab1_dtim = tab1_tim - tab1_tim[0] # dmax = np.amax(tab1_spec_plt) # dmin = np.amin(tab1_spec_plt) # tab1_spec_plt = (tab1_spec_plt - dmin) / (dmax - dmin) * 255. load_specdata(specfile) TOOLS = "pan,wheel_zoom,box_zoom,reset,save" '''create the dynamic spectrum plot''' tab1_p_dspec = figure(tools=TOOLS, webgl=config_plot['plot_config']['WebGL'], plot_width=config_plot['plot_config']['tab1']['dspec_wdth'], plot_height=config_plot['plot_config']['tab1']['dspec_hght'], x_range=(tab1_dtim[0], tab1_dtim[-1]), y_range=(tab1_freq[0], tab1_freq[-1]), toolbar_location="above") tim0_char = jdutil.jd_to_datetime((tab1_tim[0] / 3600. / 24. + 2400000.5) * 86400. / 3600. / 24.) tim0_char = tim0_char.strftime('%Y-%b-%d %H:%M:%S') + '.{}'.format(round(tim0_char.microsecond / 1e3) * 1e3)[0:4] tab1_p_dspec.axis.visible = True tab1_p_dspec.title.text = "Dynamic spectrum" tab1_p_dspec.xaxis.axis_label = 'Seconds since ' + tim0_char tab1_p_dspec.yaxis.axis_label = 'Frequency [GHz]' tab1_p_dspec.border_fill_color = "whitesmoke" tab1_p_dspec.axis.major_tick_out = 0 tab1_p_dspec.axis.major_tick_in = 5 tab1_p_dspec.axis.minor_tick_out = 0 tab1_p_dspec.axis.minor_tick_in = 3 tab1_p_dspec.axis.major_tick_line_color = "white" tab1_p_dspec.axis.minor_tick_line_color = "white" tab1_SRC_dspec = ColumnDataSource(data={'data': [tab1_spec_plt], 'xx': [tab1_dtim], 'yy': [tab1_freq]}) tab1_r_dspec = tab1_p_dspec.image(image="data", x=tab1_dtim[0], y=tab1_freq[0], dw=tab1_dtim[-1] - tab1_dtim[0],
#tau = data['tau'] #secz = data['secz'] #plt.clf() #plt.ion() #plt.figure() #plt.plot(secz,tau,'o') #plt.ylabel('tau') #plt.xlabel('secz') #plt.xlim(1.85,1.95) #plt.ylim(0.64,0.67) # make datetime vector from JD dtime = [jdutil.jd_to_datetime(jd) for jd in data['jd']] #for i in range(len(data['jd'])): # dt = np.append(dtime,jdutil.jd_to_datetime(data['jd'][i])) dates = mpl.dates.date2num(dtime) xpos = data['xpos'] ypos = data['ypos'] mx = np.median(xpos) my = np.median(ypos) plt.ion() fig = plt.figure(1,figsize=(8.5,11)) gs = gridspec.GridSpec(5, 1,wspace=0) ax1 = plt.subplot(gs[0, 0])
def jdate(day): datet = jd.jd_to_datetime(day) return datet.strftime('%Y-%m-%d')