def readimpulseresponses(filepath='../data_fs/extern/'): name = filepath + 'signal_collection_ft.npy' s = npload(name) name = filepath + 'noise_collection_ft.npy' n = npload(name) name = filepath + 'times_collection.npy' t = npload(name) name = filepath + 'frequencies_collection.npy' f = npload(name) return (s,n,f,t)
def lith_tensor(self): """h1, h2, h3 dictionary for lith volume""" return dict( h1=npload(Wolfpass.fetch_data(filename='vol_h1.vol.npy', download_if_missing=False, verbose=False)), h2=npload(Wolfpass.fetch_data(filename='vol_h2.vol.npy', download_if_missing=False, verbose=False)), h3=npload(Wolfpass.fetch_data(filename='vol_h3.vol.npy', download_if_missing=False, verbose=False)) )
def lith_tensor(self): """h1, h2, h3 dictionary for lith volume""" return dict(h1=npload( Wolfpass.fetch_data(filename='vol_h1.vol.npy', download_if_missing=False, verbose=False)), h2=npload( Wolfpass.fetch_data(filename='vol_h2.vol.npy', download_if_missing=False, verbose=False)), h3=npload( Wolfpass.fetch_data(filename='vol_h3.vol.npy', download_if_missing=False, verbose=False)))
def lith_tensor(self): """(h1, h2, h3, x0) for lith volume""" return ( npload(Wolfpass.fetch_data(filename='vol_h1.vol.npy', download_if_missing=False, verbose=False)), npload(Wolfpass.fetch_data(filename='vol_h2.vol.npy', download_if_missing=False, verbose=False)), npload(Wolfpass.fetch_data(filename='vol_h3.vol.npy', download_if_missing=False, verbose=False)), npload(Wolfpass.fetch_data(filename='vol_x0.vol.npy', download_if_missing=False, verbose=False)) )
def loadDokMatrix(filename): 'Loads a dok_matrix saved using the above saveDokMatrix' data = npload(filename) m = dok_matrix(tuple(data['shape'])) for k, v in zip(data['keys'], data['values']): m[tuple(k)] = v return m
def lith_diorite_early_data(self): """data for early diorite surface""" return npload(Wolfpass.fetch_data( filename='dist_to_borehole.lithsurf.npy', download_if_missing=False, verbose=False ))
def cu_triangles(self): """list of cu pct surface triangles""" return [ npload( Wolfpass.fetch_data(filename=prefix + '_t.cusurf.npy', download_if_missing=False, verbose=False)) for prefix in self.cu_names ]
def loadSingleTraj(fname, fformat): """loads a single trajectory depending on the format chosen""" if fformat == "npz": ar = npload(fname) return ar['arr_0'] elif fformat == "dat": return numpy.loadtxt(fname) else: raise ValueError("Invalid file format %s" % fformat)
def lith_triangles(self): """list of lithology surface triangles""" return [ npload( Wolfpass.fetch_data(filename=prefix + '_t.lithsurf.npy', download_if_missing=False, verbose=False)) for prefix in self.lith_names ]
def loadSingleTraj(fname, fformat): """loads a single trajectory depending on the format chosen""" if fformat == "npz": ar = npload(fname) return ar['arr_0'] elif fformat == "dat": return numpy.loadtxt(fname) else: raise ValueError("Invalid file format %s" % fformat)
def topo_data(self): """elevation data""" return dict( location='N', data=DataArray( title='elevation', array=npload(Wolfpass.fetch_data(filename='elevation.toposurf.npy', download_if_missing=False, verbose=False)) ) )
def store(self, batch): r""" Stores a batch on file and returns a stored batch. This method can be passed as *store* argument to create an open :class:`Sample` instance. """ #------------------------------------------------------------------------------- with self.base.open('rb+') as u: n = int(pickle.load(u)) u.seek(0) pickle.dump(n + 1, u) u.truncate() p = self.getpath(n) npsave(p, batch) return npload(p, mmap_mode='r+')
def toSample(self): r""" Returns a closed :class:`Sample` instance. """ #------------------------------------------------------------------------------- with self.base.open('rb') as u: n = int(pickle.load(u)) L = [npload(self.getpath(k), mmap_mode='r') for k in range(n)] sample = Sample(L[0].shape[1], None) content = [(r, r_.shape[1]) for r, r_ in zip(L[:-1], L[1:])] content.append((L[-1], 0)) sample.content = tuple(content) sample.closed = True return sample
def lith_data(self): """dictionary of data for lith volume""" if getattr(self, '_lith_data', None) is None: self._lith_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.vol.npy'): continue if npyfile.startswith('vol_'): continue self._lith_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)).flatten() return self._lith_data
def drill_data(self): """dictionry of drill point data""" if getattr(self, '_drill_data', None) is None: self._drill_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.point.npy'): continue if npyfile.endswith('_v.point.npy'): continue self._drill_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) return self._drill_data
def drill_data(self): """dictionry of drill point data""" if getattr(self, '_drill_data', None) is None: self._drill_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.point.npy'): continue if npyfile.endswith('_v.point.npy'): continue self._drill_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False) ) return self._drill_data
def lith_data(self): """dictionary of data for lith volume""" if getattr(self, '_lith_data', None) is None: self._lith_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.vol.npy'): continue if npyfile.startswith('vol_'): continue self._lith_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False) ).flatten() return self._lith_data
def xsect_data(self): """dictionary of cross section data""" if getattr(self, '_xsect_data', None) is None: self._xsect_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.xsurf.npy'): continue if (npyfile.endswith('_v.xsurf.npy') or npyfile.endswith('_t.xsurf.npy')): continue self._xsect_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) return self._xsect_data
def borehole_data(self): """dictionary of borehole data""" if getattr(self, '_borehole_data', None) is None: self._borehole_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.line.npy'): continue if (npyfile.endswith('_v.line.npy') or npyfile.endswith('_s.line.npy')): continue self._borehole_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) return self._borehole_data
def lith_diorite_early_data(self): """data for early diorite surface""" return dict( location='N', data=DataArray( title='dist_to_borehole', array=npload( Wolfpass.fetch_data( filename='dist_to_borehole.lithsurf.npy', download_if_missing=False, verbose=False ) ) ) )
def borehole_data(self): """dictionary of borehole data""" if getattr(self, '_borehole_data', None) is None: self._borehole_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.line.npy'): continue if (npyfile.endswith('_v.line.npy') or npyfile.endswith('_s.line.npy')): continue self._borehole_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False) ) return self._borehole_data
def xsect_data(self): """dictionary of cross section data""" if getattr(self, '_xsect_data', None) is None: self._xsect_data = dict() for npyfile in self.filenames: if not npyfile.endswith('.xsurf.npy'): continue if (npyfile.endswith('_v.xsurf.npy') or npyfile.endswith('_t.xsurf.npy')): continue self._xsect_data[npyfile.split('.')[0]] = npload( Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False) ) return self._xsect_data
def borehole_raw_data(self): """list of title/array dictionaries for borehole data""" raw_data = [] for npyfile in self.filenames: if not npyfile.endswith('.line.npy'): continue if (npyfile.endswith('_v.line.npy') or npyfile.endswith('_s.line.npy')): continue raw_data += [dict( title=npyfile.split('.')[0], array=npload(Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) )] return raw_data
def lith_data(self): """raw data for lith volume""" raw_data = [] for npyfile in self.filenames: if not npyfile.endswith('.vol.npy'): continue if npyfile.startswith('vol_'): continue raw_data += [dict( location='CC', data=DataArray( title=npyfile.split('.')[0], array=npload(Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) ) )] return raw_data
def __init__(self,fileName,evts,chans,det,simInfo, default=''): self.fileName =fileName self.det = det self.evts = evts self.chans = chans self.tz, self.tStart, self.tStop = simInfo self.trigT = self.tz self.ebounds = npload("/Users/jburgess/Research/specSim/"+eboundDict[det]+'_ebounds.npy') self.defaultRSP = default self._CreatePrimaryHeader() self._MakeEboundsEXT() self._MakeEventsEXT() self._MakeGTI() self._MakeHDUList()
def xsect_data(self): """cross section raw data""" raw_data = [] for npyfile in self.filenames: if not npyfile.endswith('.xsurf.npy'): continue if (npyfile.endswith('_v.xsurf.npy') or npyfile.endswith('_t.xsurf.npy')): continue raw_data += [dict( location='CC', data=DataArray( title=npyfile.split('.')[0], array=npload(Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) ) )] return raw_data
def drill_data(self): """drill location raw data list of dicts of 'location' and 'data' with 'title' and 'array' """ raw_data = [] for npyfile in self.filenames: if not npyfile.endswith('.point.npy'): continue if npyfile.endswith('_v.point.npy'): continue raw_data += [dict( location='N', data=DataArray( title=npyfile.split('.')[0], array=npload(Wolfpass.fetch_data(filename=npyfile, download_if_missing=False, verbose=False)) ) )] return raw_data
def xsect_triangles(self): """cross section triangles""" return npload(Wolfpass.fetch_data(filename='xsect_t.xsurf.npy', download_if_missing=False, verbose=False))
def borehole_vertices(self): """borehole line vertices""" return npload( Wolfpass.fetch_data(filename='boreholes_v.line.npy', download_if_missing=False, verbose=False))
def topo_triangles(self): """topography triangles""" return npload(Wolfpass.fetch_data(filename='topo_t.toposurf.npy', download_if_missing=False, verbose=False))
def load_trained_artefacts(filename='cartpole_artefacts.npy'): saved_artefacts = npload(filename, allow_pickle=True) return (saved_artefacts[()]['Q'], saved_artefacts[()]['successful_episode_index'])
def lith_triangles(self): """list of lithology surface triangles""" return [npload(Wolfpass.fetch_data(filename=prefix + '_t.lithsurf.npy', download_if_missing=False, verbose=False)) for prefix in self.lith_prefixes]
def lith_diorite_early_data(self): """data for early diorite surface""" return npload( Wolfpass.fetch_data(filename='dist_to_borehole.lithsurf.npy', download_if_missing=False, verbose=False))
print >> msgout, "Invalid answer: '%s'" % overwrite overwrite = raw_input( "Warning: Output file '%s' exists. Do you wish to overwrite file? (Y/N) " % args.output) if overwrite.upper() in ("Y", "YES"): print >> msgout, "Overwriting '%s'." % args.output print >> msgout, "" elif overwrite.upper() in ("N", "NO"): print >> msgout, "Operation aborted." sys.exit() else: print >> sys.stderr, "Operation aborted. Cowardly refusing to overwrite '%s'." % args.output sys.exit() brt_type = npload(infile) metadata = npload(infile) t0 = time.time() if args.input == "-": print >> msgout, "Reading data from <stdin>...", else: print >> msgout, "Reading data from '%s'..." % args.input, msgout.flush() data = npload(infile) if infile is not sys.stdin: infile.close() print >> msgout, "%.2f seconds" % (time.time() - t0) if len(data.shape) != 2:
def __init__(self): self.pathToDark = path.join("libs", "dark.npy") self.darkData = npload(self.pathToDark) self.darkScaled = self.darkData[:] self.tLastScale = -1 self.scaleTimeOut = 60 # in seconds
reu_tf_idf_npz = load_npz(reu_tf_idf_npz) reu_ix_to_val = open(os.path.join(BASE, 'reuters/matrix_ix_to_val.json')) reu_ix_to_val = jsload(reu_ix_to_val) id_to_reu = open(os.path.join(BASE, 'reuters/id_to_reu_headline.csv')) id_to_reu = read_csv(id_to_reu) reu_vocab_to_ix = open(os.path.join(BASE, 'reuters/vocab_to_ix.json')) reu_vocab_to_ix = jsload(reu_vocab_to_ix) red_vocab_to_ix = open(os.path.join(BASE, 'reddit/vocab_to_ix.json')) red_vocab_to_ix = jsload(red_vocab_to_ix) words_compressed = open(os.path.join(BASE, 'u.npy'), 'rb') words_compressed = npload(words_compressed) docs_compressed = open(os.path.join(BASE, 'v_trans.npy'), 'rb') docs_compressed = npload(docs_compressed) ATN_word_to_ix = pload(open(os.path.join(BASE, 'allthenews_vocab_ix.p'))) reddit_ix_to_val = open(os.path.join(BASE, 'reddit/matrix_ix_to_val.json')) reddit_ix_to_val = jsload(reddit_ix_to_val) date_to_id = open(os.path.join(BASE, 'reuters/date_to_id.json')) date_to_id = jsload(date_to_id) red_text = pload(open(os.path.join(BASE, 'red_ix_to_text.p'), 'rb')) print >> sys.stderr, "LOADED FILES AT THE START" # searching_message = None
def topo_triangles(self): """topography triangles""" return npload( Wolfpass.fetch_data(filename='topo_t.toposurf.npy', download_if_missing=False, verbose=False))
def topo_data(self): """elevation data""" return npload( Wolfpass.fetch_data(filename='elevation.toposurf.npy', download_if_missing=False, verbose=False))
def lith_origin(self): """x0 for lith volume""" return npload( Wolfpass.fetch_data(filename='vol_x0.vol.npy', download_if_missing=False, verbose=False))
def lith_origin(self): """x0 for lith volume""" return npload(Wolfpass.fetch_data(filename='vol_x0.vol.npy', download_if_missing=False, verbose=False))
def load_state(self, file_path): self.holdkey_matrix = npload(file_path, allow_pickle=True).copy()
def drill_vertices(self): """drill point vertices""" return npload( Wolfpass.fetch_data(filename='drill_loc_v.point.npy', download_if_missing=False, verbose=False))
def xsect_triangles(self): """cross section triangles""" return npload( Wolfpass.fetch_data(filename='xsect_t.xsurf.npy', download_if_missing=False, verbose=False))
def drill_vertices(self): """drill point vertices""" return npload(Wolfpass.fetch_data(filename='drill_loc_v.point.npy', download_if_missing=False, verbose=False))
def topo_data(self): """elevation data""" return npload(Wolfpass.fetch_data(filename='elevation.toposurf.npy', download_if_missing=False, verbose=False))
dl = Data_Loading('./total_count.npy', './mfs_3342.csv') dataset = dl.raw_data genes_list = dl.genes_list dataset = dataset.T assert dataset.shape == (94655, 32738) type_counts = [ 10085, 2612, 9232, 10479, 11213, 8385, 11953, 10209, 10224, 10263 ] dn = DataNormalization(dataset, type_counts) data_norm = dn.data true_labels = [] for ti in range(10): true_labels.extend([ti for i in range(type_counts[ti])]) repeats = 1 data = data_norm from numpy import load as npload total_genes = npload('./total_genes.npy') for i in range(repeats): for genes in genes_list[:]: g_filter = [g in genes for g in total_genes] data = data[:, g_filter] #kmeans of different metrics ce = Clustering_eval(data, true_labels, metrics='euclidean' ) # metrics: euclideanm,pearson or spearman print('genes: %d,time used: %s, nmi: %f, ari: %f, fmi: %f' % (len(genes.keys()), ce._time, ce.nmi, ce.ari, ce.fms)) data = data_norm # recover
def borehole_vertices(self): """borehole line vertices""" return npload(Wolfpass.fetch_data(filename='boreholes_v.line.npy', download_if_missing=False, verbose=False))
print('===> Loading datasets') # Load the training and testing kernels Ktrain = th.load(opt.psfTrainPath) for key in Ktrain.keys(): Ktrain[key] = Ktrain[key].float() Ktest = th.load(opt.psfTestPath) for key in Ktest.keys(): Ktest[key] = Ktest[key].float() if opt.imdbPath == "": Ntrain, Ntest = 400, 100 else: f = npload(opt.imdbPath) Ntrain = f['train_set'].shape[-1] Ntest = f['test_set'].shape[-1] del f assert(opt.numTrainImagesperPSF <= Ntrain and opt.numTestImagesperPSF <= Ntest),\ "Invalid values for one or both of numTrainImagesperPSF and numTestImagesperPSF." train_mask = getSubArrays(0,Ntrain,len(Ktrain),length=opt.numTrainImagesperPSF,dformat=lambda x:ndarray(x)) test_mask = getSubArrays(0,Ntest,len(Ktest),length=opt.numTestImagesperPSF,dformat=lambda x:ndarray(x)) train_data_loader = {} test_data_loader = {} for k in range(len(Ktrain)): train_set = BSDS_deblur(Ktrain[k],opt.stdn,random_seed=opt.data_seed,filepath=opt.imdbPath,train=True,color=opt.color,shape=(256,256),batchSize=50,mask=train_mask[k]) train_data_loader[k] = DataLoader(dataset = train_set, num_workers=opt.threads, batch_size=opt.batchSize, shuffle=True) for k in range(len(Ktest)): test_set = BSDS_deblur(Ktest[k],opt.stdn,random_seed=opt.data_seed,filepath=opt.imdbPath,train=False,color=opt.color,shape=(256,256),batchSize=50,mask=test_mask[k])
def borehole_segments(self): """borehole segment vertex indices""" return npload(Wolfpass.fetch_data(filename='boreholes_s.line.npy', download_if_missing=False, verbose=False))
def openModel(self, fDir, fName): """opens an orti file, stored in xml""" self.fileDir, self.fileName = fDir, fName self.initAll() fullName = fDir + os.sep + fName if fName + '.orti' in os.listdir(self.fileDir): fullName += '.orti' else: fullName += '.iqpht' flgArr = False f1 = file(fullName, 'r') doc = f1.read() f1.close() if 'compressdata.npz' in os.listdir(fDir): darr = npload(fDir + os.sep + 'compressdata.npz') flgArr = True dom = xdom.parseString(doc) dicts = dom.getElementsByTagName("dict") for d in dicts: dname = d.getElementsByTagName("name")[0].childNodes[0].data model, typ = dname.split('_') if (model not in self.modelList) & (model != 'dic'): continue keys = d.getElementsByTagName("key") dict1 = {} for k in keys: kname = str( k.getElementsByTagName("name")[0].childNodes[0].data) kdata = k.getElementsByTagName("content")[0].childNodes[0].data exec('dict1[kname] = ' + kdata) if typ == 'val': self.dicval[model].update(dict1) #print self.dicval[model] elif typ == 'type': self.dictype[model].update(dict1) elif typ == 'zone': self.diczone[model].setDic(dict1) elif typ == 'array' and flgArr: for k in keys: kname = str( k.getElementsByTagName("name")[0].childNodes[0].data) self.dicarray[model][kname] = darr[kname.replace('.', '')] elif typ == 'formula': self.dicformula[model].update(dict1) elif typ == 'addin': self.addin.update1(dict1) #print self.dicaddin self.addin.grd = makeGrid(self, self.dicaddin['Grid']) #print 'core 152',self.addin.grd self.makeTtable() self.Zblock = makeZblock(self) mtype = self.dicaddin['Model']['group'][:5] if mtype == 'Modfl': self.flowReader = modflowReader(fDir, fName) self.transReader = mtphtReader(fDir, fName) elif mtype == 'Min3p': self.flowReader = min3pReader(fDir, fName) self.transReader = min3pReader(fDir, fName) elif mtype == 'Opgeo': self.addin.opgeo.buildMesh() self.flowReader = ogReader(fDir, fName, 'flow') self.transReader = ogReader(fDir, fName, 'trans') elif mtype == 'Sutra': self.flowReader = sutraReader(fDir, fName) self.transReader = sutraReader(fDir, fName) self.addin.setChemType()
def cu_triangles(self): """list of cu pct surface triangles""" return [npload(Wolfpass.fetch_data(filename=prefix + '_t.cusurf.npy', download_if_missing=False, verbose=False)) for prefix in self.cu_prefixes]
"Warning: Output file '%s' exists. Do you wish to overwrite file? (Y/N) " % args.output) if overwrite.upper() in ("Y", "YES"): print >> msgout, "Overwriting '%s'." % args.output print >> msgout, "" elif overwrite.upper() in ("N", "NO"): print >> msgout, "Operation aborted." sys.exit() else: print >> sys.stderr, "Operation aborted. Cowardly refusing to overwrite '%s'." % args.output sys.exit() outfile = args.output tag = npload(infile) metadata = npload(infile) data = npload(infile) if infile is not sys.stdin: infile.close() fdata = fft(data, axis=0) if len(data.shape) != 2: print >> sys.stderr, "Expected a two-dimensional array. Got an array with shape %s instead." % ( data.shape, ) sys.exit() kind_str = { "f": "floating point", "i": "integer",
def borehole_segments(self): """borehole segment vertex indices""" return npload( Wolfpass.fetch_data(filename='boreholes_s.line.npy', download_if_missing=False, verbose=False))