def __init__(self, gpu=0, checkpoint=None): self.cuda = torch.cuda.is_available() self.gpu = gpu self.device = torch.device(f"cuda:{self.gpu}" if self.cuda else "cpu") self.model = model.Network(161 * 3, 100, 86 * 3).to(self.device) root = '/Users/ferran_2020/TFG/Neural_RH_Inversion/' print("Reading Enhanced_tau_530 - tau") tmp = io.readsav(f'{root}Enhanced_tau_530.save') self.T_tau = tmp['tempi'].reshape((86, 504 * 504)) self.Pe_tau = tmp['epresi'].reshape((86, 504 * 504)) print("Reading Enhanced_tau_530 - z") tmp = io.readsav(f'{root}Enhanced_530_optiona_rh.save') self.T_z = tmp['tg'] #.reshape((161, 504*504)) self.Pg_z = tmp['pg'] #.reshape((161, 504*504)) self.z = tmp['z'] / 1e3 import pickle filename = '/Users/ferran_2020/TFG/Neural_RH_Inversion/checkpoint_Whole.dict' with open(filename, 'rb') as f: s = pickle.load(f) f.close() self.model.load_state_dict(s[0])
def main(): postx=readsav('samples_x1.sav', python_dict=True) postt=readsav('samples_t2.sav', python_dict=True) #alpha is the intercept and beta is the slope alpha=postx['postx1']['alpha'] beta=postx['postx1']['beta']
def find_photo_z(type, file=None, file_z=None, z=None, mu=None, sigma=None): """Type: file or gauss Note: file_z implementation NOT tested. """ if type == 'file': try: pz_file = io.readsav(file) pz = pz_file['p_z'] except: pz_file = np.genfromtxt(file, dtype=None, names=True) pz = pz_file['pdf'] if file_z is not None: try: my_z = io.readsav(file_z) my_z = my_z['z'] except: my_z = np.genfromtxt(file, dtype=None, names=True) my_z = my_z['z'] else: my_z = np.arange(0, 5, .01) if np.shape(pz) != np.shape(my_z): raise ValueError("pz array and z array are different sizes!") func_my_photo_z = interpolate.interp1d(my_z, pz) my_photo_z = func_my_photo_z(z) my_photo_z = np.asarray(my_photo_z/my_photo_z.max()) elif type == 'gauss': my_photo_z = stats.norm.pdf(z, mu, sigma) my_photo_z = np.asarray(my_photo_z/my_photo_z.max()) return my_photo_z
def get_dict_from_file(date, prefix="eis3"): """ Reads an IDL .sav file containing EIS housekeeping data and returns its contents as a python dictionary. For speed, if the file has already been read, it may return the contents from a hidden memo. If the file is not found in the location specified it will attempt to download it once and save the file in the location originally specified. Parameters ---------- date: date or datetime object Date of the observation required. If the file is present in the sunpy data directory, it will be read from there, or downloaded to that location if it isn't. prefix: str file prefix (eis3 for thermal correction, fpp1 for doppler shift) """ key = '{0}_{1:%Y%m}.sav'.format(prefix, date) if key in __housekeeping_memo__: file_dict = __housekeeping_memo__[key] else: download_dir = os.path.join(sunpy.util.config._get_home(), 'EISpy', 'eispy', 'data', key) try: file_dict = readsav(download_dir, python_dict=True) except IOError: url = "http://sdc.uio.no/eis_wave_corr_hk_data/" + key urllib.urlretrieve(url, filename=download_dir) file_dict = readsav(download_dir, python_dict=True) warnings.warn( "File was not found, so it was downloaded and " + "placed at the given location", UserWarning) __housekeeping_memo__.update({key: file_dict}) return file_dict
def load_vlf(): print('LOAD VLF: Currently low flyer only') vlf12_lf = readsav(path + "Low-Flyer/efield_VLF/" + '35040_LFDSP_S1_VLF12_mvm_AaronB.sav') #Low flyer (35.040) vlf34_hf = readsav(path + "High-Flyer/efield_VLF/" + '35039_Main_VLF34B_VLF34Boosted.sav') data = {"vlf12_lf":vlf12_lf, "vlf34_hf":vlf34_hf} return data
def find_photo_z(type, file=None, file_z=None, z=None, mu=None, sigma=None): """Type: file or gauss Note: file_z implementation NOT tested. """ if type == 'file': try: pz_file = io.readsav(file) pz = pz_file['p_z'] except: pz_file = np.genfromtxt(file, dtype=None, names=True) pz = pz_file['pdf'] if file_z is not None: try: my_z = io.readsav(file_z) my_z = my_z['z'] except: my_z = np.genfromtxt(file, dtype=None, names=True) my_z = my_z['z'] else: my_z = np.arange(0, 5, .01) if np.shape(pz) != np.shape(my_z): raise ValueError("pz array and z array are different sizes!") func_my_photo_z = interpolate.interp1d(my_z, pz) my_photo_z = func_my_photo_z(z) my_photo_z = np.asarray(my_photo_z / my_photo_z.max()) elif type == 'gauss': my_photo_z = stats.norm.pdf(z, mu, sigma) my_photo_z = np.asarray(my_photo_z / my_photo_z.max()) return my_photo_z
def load_quats(loc_q_sc=os.getcwd() + '/data/q_sc.sav', loc_q_global=os.getcwd() + '/data/q_global.sav'): ''' Load all quaternions from a file location Input: loc_q_sc: location of spacecraft quaternion file q_sc.sav loc_q_global: location of global quaternions file q_global.sav Output: quats: dict contatining quaternion objects q_sc an q_global Example Usage: 1) quats = load_quats() ''' from pyquaternion import Quaternion from scipy.io import readsav q_sc = readsav(loc_q_sc, python_dict=True) for cam in list(q_sc.keys()): q_sc[cam] = Quaternion( (q_sc[cam][3], q_sc[cam][0], q_sc[cam][1], q_sc[cam][2])) q_global = readsav(loc_q_global, python_dict=True) for img in list(q_global.keys()): q_global[img] = Quaternion((q_global[img][3], q_global[img][0], q_global[img][1], q_global[img][2])) quats = {'q_sc': q_sc, 'q_global': q_global} return quats
def get_dict_from_file(date, prefix="eis3"): """ Reads an IDL .sav file containing EIS housekeeping data and returns its contents as a python dictionary. For speed, if the file has already been read, it may return the contents from a hidden memo. If the file is not found in the location specified it will attempt to download it once and save the file in the location originally specified. Parameters ---------- date: date or datetime object Date of the observation required. If the file is present in the sunpy data directory, it will be read from there, or downloaded to that location if it isn't. prefix: str file prefix (eis3 for thermal correction, fpp1 for doppler shift) """ key = '{0}_{1:%Y%m}.sav'.format(prefix, date) if key in __housekeeping_memo__: file_dict = __housekeeping_memo__[key] else: download_dir = os.path.join(sunpy.util.config._get_home(), 'EISpy', 'eispy', 'data', key) try: file_dict = readsav(download_dir, python_dict=True) except IOError: url = "http://sdc.uio.no/eis_wave_corr_hk_data/" + key urllib.urlretrieve(url, filename=download_dir) file_dict = readsav(download_dir, python_dict=True) warnings.warn("File was not found, so it was downloaded and " + "placed at the given location", UserWarning) __housekeeping_memo__.update({key: file_dict}) return file_dict
def loadfhdsav(savfile): #get the other poles pol = savfile.split('_')[-1][:2] if pol=='xx': uvfile_altpol = savfile.replace('xx','yy') pols = [0,1] elif pol=='yy': uvfile_altpol = savfile.replace('yy','xx') pols = [1,0] else: print "polarization not found in filename. skipping" raise(StandardError) if not os.path.exists(uvfile_altpol): print "pol file",uvfile_altpol,"not found. please find" raise(StandardError) #paramfile = savfile.split('_')[0]+'_params.sav' paramfile = savfile.replace('vis_%s'%(['xx','yy'][pols[0]]),'params') if not os.path.exists(paramfile): print "error: paramfile=",paramfile,"not found. please find" raise(StandardError) #weightfile = savfile.split('_')[0]+'_flags.sav' weightfile = savfile.replace('vis_%s'%(['xx','yy'][pols[0]]),'flags') if not os.path.exists(weightfile): print "error: weightfile",weightfile,"not found, please find" raise(StandardError) print "loading:",savfile uvfile = readsav(savfile) ant1 = uvfile['obs']['baseline_info'][0]['tile_a'][0]-1 print "min(ant1)=",ant1.min(),"should be 0" ant2 = uvfile['obs']['baseline_info'][0]['tile_b'][0]-1 print "max(ant2)=",ant2.max(),"should be 127" data = uvfile['vis_ptr'] #times = uvfile['obs']['baseline_info'][0]['jdate'][0] baselines = (ant2)*256+ant1 freqs = uvfile['obs']['baseline_info'][0]['freq'][0] print "loading alternate polarization",uvfile_altpol uv_altpol = readsav(uvfile_altpol) data_altpol = uv_altpol['vis_ptr'] print "loading baselines from params file:",paramfile params = readsav(paramfile) U = params['params']['uu'][0]*1e9 V = params['params']['vv'][0]*1e9 W = params['params']['ww'][0]*1e9 uvw = n.array(zip(U,V,W)) times = params['params']['time'][0] print "loading weights from :",weightfile flags = readsav(weightfile) mask = n.dstack([flags['flag_arr'][0],flags['flag_arr'][1]])==0 #the zeros are the flags #create the new fits file outdata = n.zeros((data.shape[0],data.shape[1],2)).astype(n.complex64) outdata[:,:,pols[0]] = data outdata[:,:,pols[1]] = data_altpol ant1,ant2 = i2a(ant1),i2a(ant2) return uvw,ant1,ant2,baselines,times,freqs,outdata,mask
def run_master(file_int, file_vv, db_images, db_vel, zero): print(" * MASTER : reading {0}...".format(file_int)) im = io.readsav('/net/duna/scratch1/aasensio/deepLearning/opticalFlow/database/{0}.save'.format(file_int))['int'] print(" * MASTER : reading {0}...".format(file_vv)) vel = io.readsav('/net/duna/scratch1/aasensio/deepLearning/opticalFlow/database/{0}.save'.format(file_vv)) n_timesteps, nx_orig, ny_orig = im.shape tasks = [i for i in range(n_timesteps)] task_index = 0 num_workers = size - 1 closed_workers = 0 print("*** Master starting with {0} workers".format(num_workers)) while closed_workers < num_workers: dataReceived = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=status) source = status.Get_source() tag = status.Get_tag() if tag == tags.READY: # Worker is ready, so send it a task if task_index < len(tasks): dataToSend = {'index': task_index+zero, 'image': im[task_index,0:1008,0:1008], 'vx1': vel['vx1'][task_index,0:1008,0:1008], 'vz1': vel['vz1'][task_index,0:1008,0:1008], 'vx01': vel['vx01'][task_index,0:1008,0:1008], 'vz01': vel['vz01'][task_index,0:1008,0:1008], 'vx001': vel['vx001'][task_index,0:1008,0:1008], 'vz001': vel['vz001'][task_index,0:1008,0:1008]} comm.send(dataToSend, dest=source, tag=tags.START) print(" * MASTER : sending task {0} to worker {1}".format(task_index, source), flush=True) task_index += 1 else: print("Sending termination") comm.send(None, dest=source, tag=tags.EXIT) elif tag == tags.DONE: index = dataReceived['index'] im_r = dataReceived['image'] vx1_r = dataReceived['vx1'] vz1_r = dataReceived['vz1'] vx01_r = dataReceived['vx01'] vz01_r = dataReceived['vz01'] vx001_r = dataReceived['vx001'] vz001_r = dataReceived['vz001'] db_images[index,:,:] = im_r db_vel[0,index,:,:] = vx1_r db_vel[1,index,:,:] = vz1_r db_vel[2,index,:,:] = vx01_r db_vel[3,index,:,:] = vz01_r db_vel[4,index,:,:] = vx001_r db_vel[5,index,:,:] = vz001_r print(" * MASTER : got block {0} from worker {1}".format(index, source), flush=True) elif tag == tags.EXIT: print(" * MASTER : worker {0} exited.".format(source)) closed_workers += 1 print("Master block finished") return len(tasks)
def loadfhdsav(savfile): # get the other poles pol = savfile.split("_")[-1][:2] if pol == "xx": uvfile_altpol = savfile.replace("xx", "yy") pols = [0, 1] elif pol == "yy": uvfile_altpol = savfile.replace("yy", "xx") pols = [1, 0] else: print "polarization not found in filename. skipping" raise (StandardError) if not os.path.exists(uvfile_altpol): print "pol file", uvfile_altpol, "not found. please find" raise (StandardError) # paramfile = savfile.split('_')[0]+'_params.sav' paramfile = savfile.replace("vis_%s" % (["xx", "yy"][pols[0]]), "params") if not os.path.exists(paramfile): print "error: paramfile=", paramfile, "not found. please find" raise (StandardError) # weightfile = savfile.split('_')[0]+'_flags.sav' weightfile = savfile.replace("vis_%s" % (["xx", "yy"][pols[0]]), "flags") if not os.path.exists(weightfile): print "error: weightfile", weightfile, "not found, please find" raise (StandardError) print "loading:", savfile uvfile = readsav(savfile) ant1 = uvfile["obs"]["baseline_info"][0]["tile_a"][0] - 1 print "min(ant1)=", ant1.min(), "should be 0" ant2 = uvfile["obs"]["baseline_info"][0]["tile_b"][0] - 1 print "max(ant2)=", ant2.max(), "should be 127" data = uvfile["vis_ptr"] # times = uvfile['obs']['baseline_info'][0]['jdate'][0] baselines = (ant2) * 256 + ant1 freqs = uvfile["obs"]["baseline_info"][0]["freq"][0] print "loading alternate polarization", uvfile_altpol uv_altpol = readsav(uvfile_altpol) data_altpol = uv_altpol["vis_ptr"] print "loading baselines from params file:", paramfile params = readsav(paramfile) U = params["params"]["uu"][0] * 1e9 V = params["params"]["vv"][0] * 1e9 W = params["params"]["ww"][0] * 1e9 uvw = n.array(zip(U, V, W)) times = params["params"]["time"][0] print "loading weights from :", weightfile flags = readsav(weightfile) mask = n.dstack([flags["flag_arr"][0], flags["flag_arr"][1]]) == 0 # the zeros are the flags # create the new fits file outdata = n.zeros((data.shape[0], data.shape[1], 2)).astype(n.complex64) outdata[:, :, pols[0]] = data outdata[:, :, pols[1]] = data_altpol ant1, ant2 = i2a(ant1), i2a(ant2) return uvw, ant1, ant2, baselines, times, freqs, outdata, mask
def read_pfs(spectrum_path, wvlen_soln, verbose=False, is_template=False): """ Duplicate from cdips_followup.spectools because of py27 compatability. """ if not is_template: sp = readsav(spectrum_path, python_dict=True, verbose=verbose) wvlen = readsav(wvlen_soln, python_dict=True, verbose=verbose) return sp['sp'], wvlen['w'] else: s = readsav(spectrum_path, python_dict=True, verbose=verbose) return s['star'], s['w']
def __init__(self, filename, cubeInfoFilename): """ Reads in the polarised-differential calibrated VAMPIRES data produced by the IDL pipeline :param filename: The full filename of the data file. E.g. diffdata_vega_.......idlvar """ dObj = io.readsav(filename, python_dict=False, verbose=False) self.vhvv = dObj.vhvv self.vhvverr = dObj.vhvverr self.vhvvu = dObj.vhvvu self.vhvvuerr = dObj.vhvvuerr self.blengths = dObj.blengths self.bazims = dObj.bazims self.inFilename = filename try: self.diffCP = dObj.cp self.diffCPerr = dObj.cperr self.diffCPu = dObj.cpu self.diffCPuerr = dObj.cpuerr self.BL2H_IX = dObj.BL2H_IX self.H2BL_IX = dObj.H2BL_IX self.BL2BS_IX = dObj.BL2BS_IX self.BS2BL_IX = dObj.BS2BL_IX except: print "Couldn't find diff CP data for " + filename try: self.u_coords = dObj.u_coords self.v_coords = dObj.v_coords except: # Some older files didn't have these saved self.u_coords = () self.v_coords = () del (dObj) # Get useful metadata from cubeinfo file cubeinfoObj = io.readsav(cubeInfoFilename, python_dict=False, verbose=False) self.UTCs = cubeinfoObj.olog.utc[0] self.filters = cubeinfoObj.olog.filter[0] self.ras = cubeinfoObj.olog.ra[0] self.decs = cubeinfoObj.olog.dec[0] self.mask = cubeinfoObj.olog.mask[0] self.adate = cubeinfoObj.olog.adate[0] self.emgains = cubeinfoObj.olog.emgain[0] self.mffile = cubeinfoObj.plog.mf_file[0] self.pkflux = cubeinfoObj.framestats.pkflx[0] self.totflux = cubeinfoObj.framestats.totflx[0] self.cubename = cubeinfoObj.olog.cube_fname[0][0] del (cubeinfoObj) print "Read " + filename
def load(filePath, biasOffset=True): ''' Loads data into python. Currently supports formats: 3ds, sxm, dat, nvi, nvl, mat. For 3ds and dat file types their is an optional flag to correct for bias offset that is true by default. This does not correct for a current offset, and should not be used in cases where there is a significant current offset. Note: mat files are supported as exports from STMView only. Please include the file extension in the path, e.g. 'file.3ds' Usage: data = load('file.3ds', biasOffset=True) ''' if filePath.endswith('.3ds'): if biasOffset: return _correct_bias_offset(Nanonis3ds(filePath), '.3ds') else: return Nanonis3ds(filePath) elif filePath.endswith('.sxm'): return NanonisSXM(filePath) elif filePath.endswith('.dat'): if biasOffset: return _correct_bias_offset(NanonisDat(filePath), '.dat') else: return NanonisDat(filePath) elif filePath[-3:] == 'NVI' or filePath[-3:] == 'nvi': return NISTnvi(sio.readsav(filePath)) elif filePath[-3:] == 'NVL' or filePath[-3:] == 'nvl': return NISTnvl(sio.readsav(filePath)) elif filePath.endswith('.mat'): raw_mat = matio.loadmat(filePath) mappy_dict = {} for key in raw_mat: try: mappy_dict[key] = matio.Mappy() mappy_dict[key].mat2mappy(raw_mat[key]) print('Created channel: {:}'.format(key)) except: del mappy_dict[key] print('Could not convert: {:}'.format(key)) if len(mappy_dict) == 1: return mappy_dict[mappy_dict.keys()[0]] else: return mappy_dict elif filePath.endswith('.asc'): return AsciiFile(filePath) else: raise IOError('ERR - Wrong file type.')
def __init__(self): super(Dataset, self).__init__() print("Reading database with models...") tmp = io.readsav('params.idl') params = np.transpose(tmp['params'], axes=(2,1,0)) [self.nx,self.ny,npars]=params.shape self.phys = params.reshape((self.nx*self.ny, npars)) print('Number of parameters={}'.format(npars)) # Disambiguation ibx=6 ind = np.where(self.phys[:,ibx] < 0.0)[0] if len(ind) > 0: print("Disambiguating {} profiles".format(len(ind))) self.phys[ind,ibx] = -self.phys[ind,ibx] self.phys[ind,ibx+1] = -self.phys[ind,ibx+1] normalization = np.array([1e3, 1e3, 1e3, 1e3, 1e3, 1e3, 1e3, 1e3, 1e5]) normalization_mean = np.array([5e3, 5e3, 5e3, 5e3, 5e3, 0, 0, 0, 0]) self.phys = (self.phys - normalization_mean[None,:]) / normalization[None,:] self.n_training, self.n_par = self.phys.shape print("Reading database with Stokes profiles...") tmp=io.readsav('database.prof.idl') [self.nlam,self.ny2,self.nx2]=tmp['stki'].shape self.stokes=np.zeros((self.nx2,self.ny2,self.nlam,4)) self.stokes[:,:,:,0]=np.transpose(tmp['stki'], axes=(2,1,0) ) self.stokes[:,:,:,1]=np.transpose(tmp['stkq'], axes=(2,1,0) ) self.stokes[:,:,:,2]=np.transpose(tmp['stku'], axes=(2,1,0) ) self.stokes[:,:,:,3]=np.transpose(tmp['stkv'], axes=(2,1,0) ) # print("Normalizing Stokes parameters...") #[self.nx2,self.ny2,self.nlam,nstokes]=self.stokes.shape if self.nx2 != self.nx or self.ny2 != self.ny: print('params.idl is {}x{} but database_prof.npy is {}x{}'.format(self.nx,self.ny,self.nx2,self.ny2)) sys.exit(1) self.stokes[:,:,:,1] /= self.Weights[1] self.stokes[:,:,:,2] /= self.Weights[2] self.stokes[:,:,:,3] /= self.Weights[3] print("Reshaping Stokes...") self.stokes = np.transpose(self.stokes, axes=(0,1,3,2)) self.stokes = self.stokes.reshape((self.nx*self.ny, 4 * self.nlam)) self.n_training, self.n_spectral = self.stokes.shape
def idl2python(idlsavefile): ''' PORPOISE : Read IDL save file and return a dictionary with stored values ''' returndict = io.readsav(idlsavefile) return returndict
def calc_specifics(self, Temp): """A separate method to calculate the specific line list properties based on an input T.""" if self.specs_calced == 0: #make sure we don't inadvertently try and do this twice if self.ll_name == 'HITRAN04': self.Temp = Temp self.specs_calced = 1 #lets make sure the relevant temperature is now carried around with the linelist. props = HT04_globals(self.spec, self.iso) if Temp == 296.0 and self.ll_name == 'HITRAN04': Q=props.Q296 else: Q=getQ(self.spec, self.iso, self.ll_name, Temp) E_temp = -1.0 * self.epp * c2 / Temp #print E_temp w_temp = -1.0 * self.wave * c2 / Temp #print w_temp self.strength = self.strength * (props.abund/ Q) * (np.exp(E_temp) * (1.0-np.exp(w_temp))) * apc.c.cgs.value #I have no idea why Jan multiplies by C here, but he does, so lets copy it. strengths_jan = readsav('/home/dstock/sf/idl/code/ff.xdr')
def grabdate(d): global rundir, srcdir, outdir # Limit output to one line per date. status(d) # Loop over both probes. for p in ('a', 'b'): status(p) # Nuke the run directory. Leave stdout and stderr. [ os.remove(x) for x in os.listdir(rundir) if x not in ('stdoe.txt',) ] # Create and execute an IDL script to grab position, electric field, and # magnetic field data for the day and and dump it into a sav file. out, err = spedas( idlcode(probe=p, date=d) ) # Make sure there's somewhere to store the pickles. pkldir = outdir + d.replace('-', '') + '/' + p + '/' if not os.path.exists(pkldir): os.makedirs(pkldir) # Read in the IDL output. if not os.path.exists('temp.sav'): status('X') continue else: temp = io.readsav('temp.sav') # Rewrite the data as pickles. (Pickles are Python data files. They are # reasonably efficient in terms of both storage size and load time.) for key, arr in temp.items(): with open(pkldir + key + '.pkl', 'wb') as handle: pickle.dump(arr, handle, protocol=-1) # Acknowledge successful date access. status('OK') # Move to the next line. return status()
def load(filename="sme.npy"): """ Load SME data from disk Currently supported file formats: * ".npy": Numpy save file of an SME_Struct * ".sav", ".inp", ".out": IDL save file with an sme structure * ".ech": Echelle file from (Py)REDUCE Parameters ---------- filename : str, optional name of the file to load (default: 'sme.npy') Returns ------- sme : SME_Struct Loaded SME structure Raises ------ ValueError If the file format extension is not recognized """ logging.info("Loading SME file %s", filename) ext = Path(filename).suffix if ext == ".npy": # Numpy Save file s = np.load(filename, allow_pickle=True) s = np.atleast_1d(s)[0] elif ext == ".npz": s = np.load(filename, allow_pickle=True) s = s["sme"][()] elif ext in [".sav", ".out", ".inp"]: # IDL save file (from SME) s = readsav(filename)["sme"] s = {name.casefold(): s[name][0] for name in s.dtype.names} s = SME_Struct(**s) elif ext == ".ech": # Echelle file (from REDUCE) ech = echelle.read(filename) s = SME_Struct() if hasattr(ech, "columns"): s.wind = np.cumsum([0, *np.diff(ech.columns, axis=1).ravel()]) s.wave = np.ma.compressed(ech.wave) s.spec = np.ma.compressed(ech.spec) s.uncs = np.ma.compressed(ech.sig) s.mask = np.full(s.sob.size, 1) s.wran = [[w[0], w[-1]] for w in s.wave] try: s.object = ech.head["OBJECT"] except KeyError: pass else: options = [".npy", ".sav", ".out", ".inp", ".ech"] raise ValueError( f"File format not recognised, expected one of {options} but got {ext}" ) return s
def readsnrsav(name, extsavdir=None): """read the extinction data from .sav files of Chen. return its xygrid ar data and disbins Args: name: name of target SNR, eg.'snr169' extsavdir:a path of the dir storing .sav files Returns: xgrid, ygrid: 1-d arrray, coordinates of centers of every grids ar: 1-d array with 2-d array as elements, extinction data loaded from .sav files realdis: 1-d array, distance of centers of each bin """ #set the datapath if extsavdir == None: data_file = pjoin('..', '..', 'Data', 'extin3d', '{0}extin3d015.sav'.format(name)) else: data_file = pjoin(extsavdir, '{0}extin3d015.sav'.format(name)) #read cordinate, distance and extinction for each star from sav files res = readsav(data_file, python_dict=True) res = res['exta'] xgrid = res.gl[0] ygrid = res.gb[0] ar = res.dar[0] dismo = res.dis[0] #distant modulu realdis = 10.**(dismo / 5. + 1) / 1000. return xgrid, ygrid, ar, realdis
def load_nvi(filePath): '''UNTESTED - Load NISTview image data into python. ''' nviData = sio.readsav(filePath) self = Spy() self._raw = nviData['imagetosave'] self.map = self._raw.currentdata[0] self.header = { name: self._raw.header[0][name][0] for name in self._raw.header[0].dtype.names } self.info = { 'FILENAME': self._raw.filename[0], 'FILSIZE': int(self._raw.header[0].filesize[0]), 'CHANNELS': self._raw.header[0].scan_channels[0], 'XSIZE': self._raw.xsize[0], 'YSIZE': self._raw.ysize[0], 'TEMPERATURE': self._raw.header[0].temperature[0], 'LOCKIN_AMPLITUDE': self._raw.header[0].lockin_amplitude[0], 'LOCKIN_FREQUENCY': self._raw.header[0].lockin_frequency[0], 'DATE': self._raw.header[0].date[0], 'TIME': self._raw.header[0].time[0], 'BIAS_SETPOINT': self._raw.header[0].bias_setpoint[0], 'BIAS_OFFSET': self._raw.header[0].bias_offset[0], 'BFIELD': self._raw.header[0].bfield[0], 'ZUNITS': self._raw.zunits[0], } return self
def import_cosmology(filename, structure_name="fid"): r""" Loads an icosmo cosmology from a fiducial structure stored in an idl save file into a cosmicpy cosmology. Parameters ---------- filename : str Name of the idl save from which to load the cosmology. structure_name : str, optional Name of the icosmo fiducial structure stored in the save file. Returns ------- cosmo : cosmology cosmicpy cosmology corresponding to the icosmo input. """ icosmo_file = readsav(filename) icosmo = icosmo_file.get(structure_name) h = icosmo['cosmo'][0]['h'][0] Omega_m = icosmo['cosmo'][0]['omega_m'][0] Omega_de = icosmo['cosmo'][0]['omega_l'][0] Omega_b = icosmo['cosmo'][0]['omega_b'][0] w0 = icosmo['cosmo'][0]['w0'][0] wa = icosmo['cosmo'][0]['wa'][0] tau = icosmo['cosmo'][0]['tau'][0] n = icosmo['cosmo'][0]['n'][0] sigma8 = icosmo['cosmo'][0]['sigma8'][0] cosmo = cosmicpy.cosmology(h=h, Omega_m=Omega_m, Omega_de=Omega_de, Omega_b=Omega_b, w0=w0, wa=wa, tau=tau, n=n, sigma8=sigma8) return cosmo
def makeWeightsArray(X): savfile = readsav(X) wuv = savfile['weights_uv'] wxy = n.fft.fft2(wuv) wuv = n.abs(wuv) wxy = n.abs(wxy) return wuv,wxy
def test_arrays_replicated_3d(self): pth = path.join(DATA_PATH, 'struct_pointer_arrays_replicated_3d.sav') s = readsav(pth, verbose=False) # Check column types assert_(s.arrays_rep.g.dtype.type is np.object_) assert_(s.arrays_rep.h.dtype.type is np.object_) # Check column shapes assert_equal(s.arrays_rep.g.shape, (4, 3, 2)) assert_equal(s.arrays_rep.h.shape, (4, 3, 2)) # Check values for i in range(4): for j in range(3): for k in range(2): assert_array_identical( s.arrays_rep.g[i, j, k], np.repeat(np.float32(4.), 2).astype(np.object_)) assert_array_identical( s.arrays_rep.h[i, j, k], np.repeat(np.float32(4.), 3).astype(np.object_)) assert_( np.all( vect_id(s.arrays_rep.g[i, j, k]) == id( s.arrays_rep.g[0, 0, 0][0]))) assert_( np.all( vect_id(s.arrays_rep.h[i, j, k]) == id( s.arrays_rep.h[0, 0, 0][0])))
def test_arrays_replicated(self): s = readsav(path.join(DATA_PATH, 'struct_pointer_arrays_replicated.sav'), verbose=False) # Check column types assert_(s.arrays_rep.g.dtype.type is np.object_) assert_(s.arrays_rep.h.dtype.type is np.object_) # Check column shapes assert_equal(s.arrays_rep.g.shape, (5, )) assert_equal(s.arrays_rep.h.shape, (5, )) # Check values for i in range(5): assert_array_identical( s.arrays_rep.g[i], np.repeat(np.float32(4.), 2).astype(np.object_)) assert_array_identical( s.arrays_rep.h[i], np.repeat(np.float32(4.), 3).astype(np.object_)) assert_( np.all(vect_id(s.arrays_rep.g[i]) == id(s.arrays_rep.g[0][0]))) assert_( np.all(vect_id(s.arrays_rep.h[i]) == id(s.arrays_rep.h[0][0])))
def test_scalars(self): s = readsav(path.join(DATA_PATH, 'struct_pointers.sav'), verbose=False) assert_identical(s.pointers.g, np.array(np.float32(4.), dtype=np.object_)) assert_identical(s.pointers.h, np.array(np.float32(4.), dtype=np.object_)) assert_(id(s.pointers.g[0]) == id(s.pointers.h[0]))
def test_7d(self): s = readsav(path.join(DATA_PATH, 'array_float32_pointer_7d.sav'), verbose=False) assert_equal(s.array7d.shape, (2, 1, 2, 3, 4, 3, 2)) assert_(np.all(s.array7d == np.float32(4.))) assert_( np.all(vect_id(s.array7d) == id(s.array7d[0, 0, 0, 0, 0, 0, 0])))
def test_arrays_replicated_3d(self): s = readsav(path.join(DATA_PATH, 'struct_arrays_replicated_3d.sav'), verbose=False) # Check column types assert_(s.arrays_rep.a.dtype.type is np.object_) assert_(s.arrays_rep.b.dtype.type is np.object_) assert_(s.arrays_rep.c.dtype.type is np.object_) assert_(s.arrays_rep.d.dtype.type is np.object_) # Check column shapes assert_equal(s.arrays_rep.a.shape, (4, 3, 2)) assert_equal(s.arrays_rep.b.shape, (4, 3, 2)) assert_equal(s.arrays_rep.c.shape, (4, 3, 2)) assert_equal(s.arrays_rep.d.shape, (4, 3, 2)) # Check values for i in range(4): for j in range(3): for k in range(2): assert_array_identical(s.arrays_rep.a[i, j, k], np.array([1, 2, 3], dtype=np.int16)) assert_array_identical( s.arrays_rep.b[i, j, k], np.array([4., 5., 6., 7.], dtype=np.float32)) assert_array_identical( s.arrays_rep.c[i, j, k], np.array([np.complex64(1 + 2j), np.complex64(7 + 8j)])) assert_array_identical( s.arrays_rep.d[i, j, k], np.array([b"cheese", b"bacon", b"spam"], dtype=object))
def __init__(self, sav_fileloc = '/home/wilcoxr/m3dc1/profile_savs/dens_xsection.sav', linfac = -2.5 * 4 / np.pi): """ I'm only saving this in case I have trouble making class M3DC1_xsection backwards compatible Requires previously saved IDL .sav file containing profile variables I generate this data with this (on venus): /u/wilcoxr/idl/load_m3dc1_single_n_fullxsection.pro ** This is only OK to do simply because I'm using the toroidal location at the peak of the density perturbation! Any other location requires more processing For original plot_dens_xsection: sav_fileloc = '/home/wilcoxr/m3dc1/profile_savs/dens_xsection.sav', linfac = -2.5*4/np.pi """ idlsave_dict = si.readsav(sav_fileloc, python_dict = True, verbose = False) # fields = ['te','den','ti','pe','p'] (usually ordered like this in arrays) self.fields = idlsave_dict[ 'fields'] # fields that are in the IDL sav file, and which index corresponds to which self.r = idlsave_dict['r'] # grid locations (401) self.z = idlsave_dict['z'] self.dens_pert = linfac * np.real( idlsave_dict['res']) # Not correct unless just looking at the peak of the dens pert
def sav_to_fits(savfile): """ Simple method to convert the DEIMOS .sav files, which contain the optical model maps, to .fits files. ToDO: This is specific for keck_deimos `static_calib` data, since the path is explicitly mentioned. If needed, this method could be generalized. TODO: move it to `pypeit.io`. Args: savfile: path to the .sav file Returns: Save the content of the .sav file into a .fits file in `data/static_calibs/keck_deimos/` """ savfile_name = os.path.splitext(os.path.basename(savfile))[0] sav = readsav(savfile, python_dict=True) list_keys = list(sav.keys()) for k in list_keys: if type(sav[k]) is not np.ndarray: sav[k] = np.asarray([sav[k]]) to_path = resource_filename('pypeit', 'data/static_calibs/keck_deimos/') io.write_to_fits(sav, to_path + savfile_name + '.fits', overwrite=True) return
def grabdate(d): global rundir, srcdir, outdir # Limit output to one line per date. status(d) # Loop over both probes. for p in ('a', 'b'): status(p) # Nuke the run directory. Leave stdout and stderr. [os.remove(x) for x in os.listdir(rundir) if x not in ('stdoe.txt', )] # Create and execute an IDL script to grab position, electric field, and # magnetic field data for the day and and dump it into a sav file. out, err = spedas(idlcode(probe=p, date=d)) # Make sure there's somewhere to store the pickles. pkldir = outdir + d.replace('-', '') + '/' + p + '/' if not os.path.exists(pkldir): os.makedirs(pkldir) # Read in the IDL output. if not os.path.exists('temp.sav'): status('X') continue else: temp = io.readsav('temp.sav') # Rewrite the data as pickles. (Pickles are Python data files. They are # reasonably efficient in terms of both storage size and load time.) for key, arr in temp.items(): with open(pkldir + key + '.pkl', 'wb') as handle: pickle.dump(arr, handle, protocol=-1) # Acknowledge successful date access. status('OK') # Move to the next line. return status()
def read_padsav(file_name, disable_UserWarnings=True): """ Reads data from an idl_dict object into XPadDataItem objects Parameters ---------- file_name : str Path to XPad*.padsav file Returns ------- items : list A list of XPadDataItems """ warning_action = "default" if (disable_UserWarnings): warning_action = "ignore" with catch_warnings(): simplefilter(warning_action, UserWarning) idl_dict = readsav(file_name) return parse_padsav(idl_dict)
def __init__(self,infile): mypath=os.getcwd() if mypath.find('/Users/rfinn') > -1: print "Running on Rose's mac pro or laptop" homedir='/Users/rfinn/' elif mypath.find('Users/kellywhalen') > -1: print "Running on Kelly's Laptop" homedir='/Users/kellywhalen/Github/Virgo/' cefile=readsav(infile) self.nulnu_iras25=cefile['nulnu_iras25'] self.nulnu_iras100=cefile['nulnu_iras100'] self.nulnu_iras12=cefile['nulnu_iras12'] self.nulnu_iras60=cefile['nulnu_iras60'] self.nulnuinlsun=cefile['nulnuinlsun'] self.lir_sanders=cefile['lir_sanders'] self.lir=cefile['lir'] self.nulnu_lw3=cefile['nulnu_lw3'] self.nulnu_lw2=cefile['nulnu_lw2'] self.lamb=cefile['lambda'] # # convert all to double-precision arrays # self.lamb=array(self.lamb,'d') self.nulnu_iras25=array(self.nulnu_iras25,'d') self.nulnu_iras100=array(self.nulnu_iras100,'d') self.nulnu_iras12=array(self.nulnu_iras12,'d') self.nulnu_iras60=array(self.nulnu_iras60,'d') self.nulnuinlsun=array(self.nulnuinlsun,'d') self.lir_sanders=array(self.lir_sanders,'d') self.lir=array(self.lir,'d') self.nulnu_lw3=array(self.nulnu_lw3,'d') self.nulnu_lw2=array(self.nulnu_lw3,'d')
def dgplot1(ax1, snrNum, level, lC, bC, hl, num): # 只画消光数据 # 整理消光数据 res = readsav('../../data/map2018/IDLs/extin3d015{0}.sav'.format(snrNum)) midMum = res.exta.dis[0] # 距离间隔的中间值,距离模数 gl = res.exta.gl[0] # 网点的银经 gb = res.exta.gb[0] # 网点的银维 # debr = np.stack(res.exta.debr[0].dar) # 区间内的E(B-V)h值,三维数组,23层,每层是一个二维矩阵 degk = np.stack(res.exta.degk[0].dar) AG = degk # + 1.987 * dehk # G波段消光 disBins = [[i - 0.125, i + 0.125] for i in midMum] AG_new = np.zeros([int(AG.shape[0] / 2), AG.shape[1], AG.shape[2]]) for i in range(int(AG.shape[0] / 2)): AG_new[i, :, :] = np.sum(AG[2 * i:2 * i +2, :, :], axis=0) AG = AG_new # 绘制消光/距离等值线图 zmax = np.percentile(AG, 98) levs = zmax / level * (np.arange(level) + 1)[1:] extin = gaussian_filter(AG[num, :, :], 0.68) cont = ax1.contour(gl, gb, extin, levs) ax1.set_xlim(lC + hl, lC - hl) ax1.set_ylim(bC - hl, bC + hl) tit = r'{0:4.2f}-{1:4.2f}'.format(disBins[num][0] * 2, disBins[num][1] * 2) ax1.text(0.02, 1.05, tit, va='top', ha='left', transform=ax1.transAxes, color='red', fontsize=12, zorder=10) # 调整刻度 ax1.xaxis.set_major_locator(ticker.MultipleLocator(1)) ax1.xaxis.set_minor_locator(ticker.MultipleLocator(0.2)) ax1.yaxis.set_major_locator(ticker.MultipleLocator(1)) ax1.yaxis.set_minor_locator(ticker.MultipleLocator(0.2)) ax1.tick_params(labelsize=10) # 调整xylabel ax1.set_xlabel('Galactic Latitude', fontsize=12) ax1.set_ylabel('Galactic Longitude', fontsize=12)
def xidl_arcspec(xidl_file, slit): xidl_dict = readsav(xidl_file) if xidl_dict['archive_arc'].ndim == 2: nspec = xidl_dict['archive_arc'].shape[0] npix = xidl_dict['archive_arc'].shape[1] else: npix = xidl_dict['archive_arc'].shape[0] # This is the best one (well-centered) calib = xidl_dict['calib'][slit] # Generate the wavelengths if calib['FUNC'] == b'CHEBY': wv_air = cheby_val(calib['FFIT'], np.arange(npix), calib['NRM'], calib['NORD']) elif calib['FUNC'] == b'POLY': wv_air = poly_val(calib['FFIT'], np.arange(npix), calib['NRM']) wv_vac = airtovac(wv_air * units.AA) if xidl_dict['archive_arc'].ndim == 2: spec = xidl_dict['archive_arc'][slit] else: spec = xidl_dict['archive_arc'] # Flip to blue to red? if wv_vac[1] < wv_vac[0]: wv_vac = wv_vac[::-1] spec = spec[::-1] # Return return wv_vac.value, spec
def __init__(self, filename): idl = readsav(filename) idlout = idl['output'] self.x = idlout['x'][0] self.y = idlout['y'][0] self.z = idlout['z'][0] self.vx = idlout['vx'][0] self.vy = idlout['vy'][0] self.vz = idlout['vz'][0] self.frac = idlout['frac'][0] self.time = idlout['time'][0] self.index = idlout['index'][0] index = sorted(list(set(idlout['index'][0]))) x0 = [idlout['x0'][0][idlout['index'][0] == i][0] for i in index] y0 = [idlout['z0'][0][idlout['index'][0] == i][0] for i in index] z0 = [idlout['y0'][0][idlout['index'][0] == i][0] for i in index] vx0 = [idlout['vx0'][0][idlout['index'][0] == i][0] for i in index] vy0 = [idlout['vy0'][0][idlout['index'][0] == i][0] for i in index] vz0 = [idlout['vz0'][0][idlout['index'][0] == i][0] for i in index] self.x0 = np.array(x0) self.y0 = np.array(y0) self.z0 = np.array(z0) self.vx0 = np.array(vx0) self.vy0 = np.array(vy0) self.vz0 = np.array(vz0) del idlout
def z_from_photo_z(photo_z_file, n, my_z_array=None): my_p_z = io.readsav(photo_z_file) pz = my_p_z['p_z'] if my_z_array is None: z = np.arange(0, 5, .01) else: z = my_z_array if np.shape(pz) != np.shape(z): raise ValueError("p_z array and z array are different sizes") dz = z[1] - z[0] pz /= (dz * pz).sum() ecdf = np.cumsum(pz * dz) cdf = interpolate.interp1d(z, ecdf) def func(x, *args): my_cdf = args[0] cdf = args[1] return abs(my_cdf - cdf(x)) out_z = [] for i in range(n): my_cdf = np.random.uniform(0, 1) my_z = optimize.fmin(func, (1.5), args=(my_cdf, cdf), disp=0) out_z.append(my_z[0]) out_z = np.asarray(out_z) return out_z
def __init__(self, root): self.root = root self.noise = noise self.batch_size = 256 self.dataFile = "/net/duna/scratch1/aasensio/deepLearning/milne/database/database_6301_hinode_1component.h5" f = h5py.File(self.dataFile, 'r') self.pars = f.get("parameters") self.lower = np.min(self.pars, axis=0) self.upper = np.max(self.pars, axis=0) f.close() self.root_hinode = "/net/nas4/fis/aasensio/scratch/HINODE/SUNSPOT/" self.label_files = ["sunspot_stokesI_512x512.sav", "sunspot_stokesQ_512x512.sav", "sunspot_stokesU_512x512.sav", "sunspot_stokesV_512x512.sav"] self.std_values = np.load('{0}_normalization.npy'.format(self.root)) labels_data = ['data_ii', 'data_qq', 'data_uu', 'data_vv'] self.stokes = np.zeros((512,512,50,4)) for i in range(4): print("Reading file {0}".format(self.label_files[i])) stokes = io.readsav("/net/nas4/fis/aasensio/scratch/HINODE/SUNSPOT/{0}".format(self.label_files[i]))[labels_data[i]] if (i == 0): mean_stokesi = np.mean(stokes[400:500,0:100,0]) stokes = stokes[:,:,0:50] / mean_stokesi self.stokes[:,:,:,i] = stokes / self.std_values[None,None,:,i] self.stokes = self.stokes.reshape((512*512,50,4))
def get_sav_data(file_name, name): readed = scio.readsav(file_name, idict=None, python_dict=False, uncompressed_file_name=None, verbose=False) return (readed['{}'.format(name)])
def read_match_file(file, *args, **kwargs): try: match = readsav(file)['match'] except Exception as e: raise Exception("cannot read match data from file: %s" % (file, )) from e return match, 'ROTSE1'
def read_wic_si_data(dtm, read_si=True, read_wic=False): """ Reads IMAGE SI and WIC image data """ from scipy.io import readsav import glob import numpy as np if read_si: fname_image = "../from_harald_frey/IMF12LSI_2002_0318_" + dtm.strftime( "%H%M") + "*.sav" if read_wic: fname_image = "../from_harald_frey/IMFHWIC_2002_0318_" + dtm.strftime( "%H%M") + "*.sav" fname_image = glob.glob(fname_image)[0] dat = readsav(fname_image, python_dict=True, verbose=False) data_dict = {} # Note: the data is originally saved in IDL # and IDL considers the first dimension to be the column. # Therefore, we do the transpose to fix it. Also we need to # rotate the image so that the dayside is on the top # data_dict["mlt_img"] = dat['imageinfo'].mlt_img[0] # data_dict["image"] = dat['imageinfo'].image[0] # data_dict["mlat"] = dat['imageinfo'].mlat[0] # data_dict["mlon"] = dat['imageinfo'].mlon[0] # data_dict["mlt"] = dat['imageinfo'].mlt[0] data_dict["mlt_img"] = np.flip(dat['imageinfo'].mlt_img[0], axis=0) data_dict["image"] = np.flip(dat['imageinfo'].image[0], axis=0) data_dict["mlat"] = np.flip(dat['imageinfo'].mlat[0], axis=0) data_dict["mlon"] = np.flip(dat['imageinfo'].mlon[0], axis=0) data_dict["mlt"] = np.flip(dat['imageinfo'].mlt[0], axis=0) return fname_image, data_dict
def traceLength(inst): """ For fine-tuning the trace lengths in the contamVerify output figures""" # Getting example trace to calculate rough estimate of trace lengths if 'NIRCam' in inst: FILE = 'rot_o1_6000.0.fits' elif 'MIRI' in inst: FILE = 'LOWbg_6000.0.fits' elif 'NIRISS' in inst: FILE = 'modelOrder12_teff6000.sav' trFile = os.path.join(TRACES_PATH, inst.replace(' ', '_'), FILE) if 'NIRCam' in inst: trFile = trFile.replace('NIRCam', 'NIRCam_F444W') trData = readsav(trFile)['modelo12'] if 'NIRISS' in inst \ else fits.getdata(trFile, 1) trData = trData[0] ax = 1 if 'NIRCam' in inst else 0 peak = trData.max() # the length of the trace targ_trace_start = np.where(trData > 0.0001 * peak)[ax].min() targ_trace_stop = np.where(trData > 0.0001 * peak)[ax].max() return targ_trace_start, targ_trace_stop
def load_telem(datestring, filenum, nirc2_mjd): """ Loads and aggregates telemetry info from one file """ acceptable_dt = .0001 # precision of mjd match (~10 seconds or so) # Telemetry file will match: telem_pattern = f"/g/lu/data/keck_telemetry/{datestring}*/**/n?{filenum}_*.sav" # Get all matching files telem_files = glob.glob(telem_pattern, recursive=True) if len(telem_files) == 0: # No matches return for telem_file in telem_files: # Read telemetry file telem = readsav(telem_file) # Get mjd telem_mjd = get_telem_mjd(telem) if np.abs(telem_mjd-nirc2_mjd) < acceptable_dt: # Get mean and std of rms residuals rms_mean = np.mean(telem.a.residualrms[0][0]) rms_std = np.std(telem.a.residualrms[0][0]) return telem_file, telem_mjd, rms_mean, rms_std del telem return None
def _get_dusty_array(y_window, x_window): """ Returns the sliced array of dusty pixels """ url = darts + 'data/cal/dp/dusty_pixels.sav' http_down = urllib.urlretrieve(url) dusties = readsav(http_down[0]).dp_data return dusties[y_window[0]:y_window[1], x_window[0]: x_window[1]]
def __init__(self, root, output, name_of_variable): # Only allocate needed memory config = tf.ConfigProto() config.gpu_options.allow_growth=True session = tf.Session(config=config) ktf.set_session(session) self.root = root self.nx = 576 self.ny = 576 self.n_times = 2 self.n_filters = 64 self.batch_size = 1 self.n_conv_layers = 20 self.stride = 1 self.skip_frequency = 2 self.n_frames = 1 self.output = output self.name_of_variable = name_of_variable telescope_radius = 0.5 * 0.965 * u.meter pixel_size = 0.02759 * u.arcsec / u.pixel fov = 1152 * u.pixel lambda0 = 500 * u.nm imax = imax_degradation(telescope_radius, pixel_size, fov) imax.compute_psf(lambda0) res = io.readsav('/net/viga/scratch1/deepLearning/opticalFlow/mancha/c3d_1152_cont4_4bin_012000_continuum.sav')['continuum'] self.images = np.zeros((2,576,576), dtype='float32') # 576 pixels are obtained by resampling 1152 pixels of 0.02759 "/px to 0.0545 "/px for IMaX self.images[0,:,:] = congrid.resample(imax.apply_psf(res[0,:,:]), (576, 576)) self.images[1,:,:] = congrid.resample(imax.apply_psf(res[1,:,:]), (576, 576)) res = io.readsav('/net/viga/scratch1/deepLearning/opticalFlow/mancha/c3d_1152_cont4_4bin_012000.isotau.sav') self.vx = np.zeros((3,576,576), dtype='float32') self.vy = np.zeros((3,576,576), dtype='float32') for i in range(3): self.vx[i,:,:] = congrid.resample(imax.apply_psf(res['vx'][i,:,:]), (576, 576)) self.vy[i,:,:] = congrid.resample(imax.apply_psf(res['vy'][i,:,:]), (576, 576))
def find_sol_mask(shotnr, frame_info=None, rz_array=None, datadir='/Users/ralph/source/blob_tracking/test_data'): """ Returns a mask for the pixels in between the separatrix and the LCFS. """ s = readsav('%s/separatrix.sav' % (datadir), verbose=False) return ((s['rmid'].reshape(64, 64) > s['rmid_sepx']) & (s['rmid'].reshape(64, 64) < s['rmid_lim']))
def get_kordopatis_comparisons(): data = readsav(os.path.join(DATA_PATH, "RAVE_DR5_calibration_data.save")) return Table(data={ "TEFF": data["calibration_data"]["TEFF"][0], "LOGG": data["calibration_data"]["LOGG"][0], "FEH": data["calibration_data"]["FEH"][0], "REF": data["calibration_data"]["REF"][0], "Name": [each.strip() for each in data["calibration_data"]["DR5_OBS_ID"][0]] })
def wotta16(): """ Generate sys files from IDL save files Returns ------- """ from scipy.io import readsav # Non-excluded all = readsav(pyigm_path+'/data/LLS/Literature/wotta16_final.save') # Build Lehner+13 assert False # Need RA/DEC
def fetch_meta(fhd_run, obsids=None): ''' Return meta data needed for the FHD deconvolved source components. Parameters ---------- fhd_run: string The name identifier of the FHD run, e.g. \'pac_decon_eor1_June2016\'. obsids: list-like, optional Obsids (as strings) to fetch data from. Defaults to all deconvolved. ''' decon_dir='%sfhd_%s/deconvolution/'%(fhd_base(),fhd_run) meta_dir='%sfhd_%s/metadata/'%(fhd_base(),fhd_run) if obsids is None: obsids = fp.get_obslist(decon_dir) meta = {'clustered':False} for o in obsids: params = readsav(decon_dir+o+'_fhd_params.sav')['fhd_params'] metaobs = readsav('%s%s_obs.sav'%(meta_dir,o))['obs'] meta[o] = {'n_iter':params.n_iter[0],'det_thresh':params.detection_threshold[0],'beam_thresh':params.beam_threshold[0],'max_bl':metaobs.max_baseline[0],'freq':metaobs.freq_center[0],'degpix':metaobs.degpix[0]} meta[o]['beam_width'] = meta[o]['max_bl']**-1 * 180./np.pi return meta
def makeVisArray(X): #savfile = [] #vis_array = [] savfile = readsav(X) for m in X.split('_'): if m == 'model': vis_array = savfile['vis_model_ptr'] print 'Model found!' break if m != 'model': vis_array = savfile['vis_ptr'] try: flag = readsav(X.split('_')[0]+'_flags.sav') X.split('_')[0]+'_flags.sav' except IOError: flag = readsav(('_').join(X.split('_')[0:3])+'_flags.sav') ('_').join(X.split('_')[0:3])+'_flags.sav' print flag.keys() flag = flag['flag_arr'][0] #vis_array[flag==1] = 0 #vis_array[flag==1] = 0 print n.max(vis_array) #vis_array[flag==-1]=0 obs = savfile['obs'] times = obs['baseline_info'][0]['JDATE'][0] #ants1 = obs['baseline_info'][0]['TILE_A'][0] #ants2 = obs['baseline_info'][0]['TILE_B'][0] #print n.where(vis_array[5000,:]==0)[0].shape ntimes = len(times) nbls = obs['NBASELINES'][0] time_order = n.argsort(times) ant1,ant2 = options.baseline.split('_') ind1 = (obs[0]['baseline_info']['tile_a'][0]==int(ant1)).astype(int) ind2 = (obs[0]['baseline_info']['tile_b'][0]==int(ant2)).astype(int) intersect = ((ind1+ind2)/2).astype(bool) print intersect.max() bsl_array = vis_array[intersect] print bsl_array.shape bsl_array = bsl_array[time_order] return bsl_array
def load_nvl(filePath): '''UNTESTED - Load NISTview layer data into python. ''' nvlData = sio.readsav(filePath) self = Spy() self._raw = nvlData['savestructure'] self.en = self._raw.energies[0] self.map = self._raw.fwddata[0] self.ave = [np.mean(layer) for layer in self.map] self.header = {name:self._raw.header[0][name][0] for name in self._raw.header[0].dtype.names} for name in self._raw.dtype.names: if name not in self.header.keys(): self.header[name] = self._raw[name][0] return self
def makeespec(name, options=None): data = readsav(name+".sav") try: return _espec_v4(data, options) except KeyError: try: print("Reading version 4 failed. Falling back on version 3.") return _espec_v3(data, options) except KeyError: try: print("Reading version 3 failed. Falling back on version 2.") return _espec_v2(data, identify=_ident_any) except IndexError: print("Reading version 2 failed. Falling back on version 1.") return _espec_v1(data)
def read_rave_data(filename): inputf = readsav(filename) items = inputf.items() data = items[0][1] wl = data['lambda'][0] # assuming they're all the same... sp = data['obs_sp'] # (75437, 839) test_flux = np.zeros((len(sp), len(sp[0]))) for jj in xrange(len(sp)): test_flux[jj, :] = sp[jj] snr = np.array(data['snr']) test_ivar = (snr[:, None] / test_flux)**2 bad = np.logical_or(np.isnan(test_ivar), np.isnan(test_flux)) test_ivar[bad] = 0. test_flux[bad] = 0. return (test_flux, test_ivar, wl)
def read_tr_data(): inputf = readsav('RAVE_DR4_calibration_data.save') items = inputf.items() data = items[0][1] tr_flux = data['spectrum'][0].T # shape (807, 839) = (nstars, npix) npix = tr_flux.shape[1] nstars = tr_flux.shape[0] teff = data['teff'][0] # length 807 logg = data['logg'][0] # length 807 feh = data['feh'][0] tr_label = np.vstack((teff, logg, feh)).T snr = np.zeros(nstars) snr.fill(100) # a guess for what the SNR could be tr_ivar = (snr[:,None]/tr_flux)**2 return tr_flux, tr_ivar, tr_label
def validation_generator(self): f = io.readsav(self.observations) out = f[self.name_of_variable] self.median_i = np.median(out[:,100:-100,100:-100]) input_validation = np.zeros((self.batch_size,self.nx,self.ny,2), dtype='float32') while 1: for i in range(self.n_frames): print('{0}/{1}'.format(i,self.n_frames)) input_validation[:,:,:,0] = out[i*self.batch_size:(i+1)*self.batch_size,100:100+self.nx,100:100+self.ny] / self.median_i input_validation[:,:,:,1] = out[i*self.batch_size+1:(i+1)*self.batch_size+1,100:100+self.nx,100:100+self.ny] / self.median_i yield input_validation f.close()
def losint_image(filein,fileout,a,e,i,asc,wbar,meanlong, nlon=50,nlat=17,lon1=35.,lon2=60.,lat1=-8.5,lat2=8.5,nstep=200): if isfile(fileout): print("ERROR: Output file exists already, exiting.") return # run IDL to generate a save file args = 'ring_image,"'+filein+'","'+fileout+'",'+str(a)+','+str(e)+','+str(i)+','+str(asc)+','+str(wbar)+','+str(meanlong)+',lon1='+str(lon1)+',lon2='+str(lon2)+',lat1='+str(lat1)+',lat2='+str(lat2)+',nlon='+str(nlon)+',nlat='+str(nlat) p1 = Popen(['echo',args],stdout=PIPE) p2 = Popen('idl',stdin=p1.stdout,stdout=PIPE) p1.stdout.close() aout = p2.communicate() print(aout) # grab desired output from save file and then delete it out = readsav(fileout) remove(fileout) return (out.im,out.lons,out.lats)
def read_sourcelist(fhdsav, tag='component_array'): ''' Get the list of obsids with deconvolution ouput. Parameters ---------- fhdsav: string fhd_run: string Full path to the IDL save file containing a source list structure. tag: string, optional The tag name of the source list in the IDL structure. Defaults to \'component_array\'. ''' cat = readsav(fhdsav)[tag] items = [cat.id, cat.x, cat.y, cat.ra, cat.dec, np.vstack(cat.flux).T['i'][0], cat.gain,cat.alpha, cat.freq, cat.flag] items = [item.astype(np.float64) for item in items] cat = dict(zip(['id','x','y','ra','dec','flux','gain','alpha','freq', 'flag'],items)) return cat
def read_geo(orbit): """Output data from a GEO file created from IDL/GDL Arguments --------- orbit : string orbit number """ fil = geo_path+orbit.zfill(7)+'_001.GEO.sav' a = readsav(fil) d = {'x': a.geo.x.item(), 'j2000': a.geo.j2000.item(), 'km': a.geo.km.item(), 'lon': a.geo.lon.item(), 'lat': a.geo.lat.item(), 'sza': a.geo.sza.item(), 'roll': a.geo.roll.item(), 'az_res': a.geo.az_res.item(), 'n_pre': a.geo.n_pre.item(), } return pd.DataFrame(d)
def predict_validation(self): print("Predicting validation data...") tmp = np.load('/net/duna/scratch1/aasensio/deepLearning/opticalFlow/database/normalization.npz') min_i, max_i, min_v, max_v = tmp['arr_0'], tmp['arr_1'], tmp['arr_2'], tmp['arr_3'] f = io.readsav(self.observations) out = f[self.name_of_variable] self.median_i = np.median(out[:,100:-100,100:-100]) input_validation = np.zeros((1,self.nx,self.ny,2), dtype='float32') input_validation[0,:,:,0] = out[0:1,100:100+self.nx,100:100+self.ny] / self.median_i input_validation[0,:,:,1] = out[1:2,100:100+self.nx,100:100+self.ny] / self.median_i # ff = io.readsav(self.observations) # im = ff['cont'] # x = np.arange(self.nx) # y = np.arange(self.ny) start = time.time() out = self.model.predict_generator(self.validation_generator(), self.n_frames, max_q_size=1) end = time.time() print("Prediction took {0} seconds...".format(end-start)) fun = ktf.function([self.model.layers[0].input],[self.model.layers[1].output]) output = np.squeeze(fun([input_validation])[0][0,200:300,200:300,:]).reshape((100,100,8,8)) f, ax = pl.subplots(nrows=2, ncols=2, figsize=(12,12)) ax[0,0].imshow(output[:,:,0,0] / np.median(output[:,:,0,0])) ax[0,1].imshow(output[:,:,4,0] / np.median(output[:,:,4,0])) ax[1,0].imshow(output[:,:,3,4] / np.median(output[:,:,3,4])) ax[1,1].imshow(output[:,:,2,2] / np.median(output[:,:,2,2])) pl.show() # stop()
def _try_download_nearest_cal(date, pix_type, detector, top_bot, left_right): """ Tries to download the requested calibration data, looking for up to one month before and after to do so. """ key = _construct_hot_warm_pix_url(date, pix_type, detector, top_bot, left_right) if key in __pix_memo__: return __pix_memo__[key] dates = _get_cal_dates(pix_type) dates.sort(key=lambda d: d - date if d > date else date - d) # dates is now a sorted list of the dates closest to the specified date for cal_date in dates: url = _construct_hot_warm_pix_url(cal_date, pix_type, detector, top_bot, left_right) http_response = urllib.urlopen(url) http_response.close() if http_response.code == 200: # HTTP OK http_down = urllib.urlretrieve(url) arr = readsav(http_down[0]).ccd_data __pix_memo__[key] = arr return arr