def readimpulseresponses(filepath='../data_fs/extern/'):
name = filepath + 'signal_collection_ft.npy'
s = npload(name)
name = filepath + 'noise_collection_ft.npy'
n = npload(name)
name = filepath + 'times_collection.npy'
t = npload(name)
name = filepath + 'frequencies_collection.npy'
f = npload(name)
return (s,n,f,t)
def lith_tensor(self):
"""h1, h2, h3 dictionary for lith volume"""
return dict(
h1=npload(Wolfpass.fetch_data(filename='vol_h1.vol.npy',
download_if_missing=False,
verbose=False)),
h2=npload(Wolfpass.fetch_data(filename='vol_h2.vol.npy',
download_if_missing=False,
verbose=False)),
h3=npload(Wolfpass.fetch_data(filename='vol_h3.vol.npy',
download_if_missing=False,
verbose=False))
)
def lith_tensor(self):
"""h1, h2, h3 dictionary for lith volume"""
return dict(h1=npload(
Wolfpass.fetch_data(filename='vol_h1.vol.npy',
download_if_missing=False,
verbose=False)),
h2=npload(
Wolfpass.fetch_data(filename='vol_h2.vol.npy',
download_if_missing=False,
verbose=False)),
h3=npload(
Wolfpass.fetch_data(filename='vol_h3.vol.npy',
download_if_missing=False,
verbose=False)))
def lith_tensor(self):
"""(h1, h2, h3, x0) for lith volume"""
return (
npload(Wolfpass.fetch_data(filename='vol_h1.vol.npy',
download_if_missing=False,
verbose=False)),
npload(Wolfpass.fetch_data(filename='vol_h2.vol.npy',
download_if_missing=False,
verbose=False)),
npload(Wolfpass.fetch_data(filename='vol_h3.vol.npy',
download_if_missing=False,
verbose=False)),
npload(Wolfpass.fetch_data(filename='vol_x0.vol.npy',
download_if_missing=False,
verbose=False))
)
def loadDokMatrix(filename):
'Loads a dok_matrix saved using the above saveDokMatrix'
data = npload(filename)
m = dok_matrix(tuple(data['shape']))
for k, v in zip(data['keys'], data['values']):
m[tuple(k)] = v
return m
def lith_diorite_early_data(self):
"""data for early diorite surface"""
return npload(Wolfpass.fetch_data(
filename='dist_to_borehole.lithsurf.npy',
download_if_missing=False,
verbose=False
))
def cu_triangles(self):
"""list of cu pct surface triangles"""
return [
npload(
Wolfpass.fetch_data(filename=prefix + '_t.cusurf.npy',
download_if_missing=False,
verbose=False)) for prefix in self.cu_names
]
def loadSingleTraj(fname, fformat):
"""loads a single trajectory depending on the format chosen"""
if fformat == "npz":
ar = npload(fname)
return ar['arr_0']
elif fformat == "dat":
return numpy.loadtxt(fname)
else:
raise ValueError("Invalid file format %s" % fformat)
def lith_triangles(self):
"""list of lithology surface triangles"""
return [
npload(
Wolfpass.fetch_data(filename=prefix + '_t.lithsurf.npy',
download_if_missing=False,
verbose=False))
for prefix in self.lith_names
]
def loadSingleTraj(fname, fformat):
"""loads a single trajectory depending on the format chosen"""
if fformat == "npz":
ar = npload(fname)
return ar['arr_0']
elif fformat == "dat":
return numpy.loadtxt(fname)
else:
raise ValueError("Invalid file format %s" % fformat)
def topo_data(self):
"""elevation data"""
return dict(
location='N',
data=DataArray(
title='elevation',
array=npload(Wolfpass.fetch_data(filename='elevation.toposurf.npy',
download_if_missing=False,
verbose=False))
)
)
def store(self, batch):
r"""
Stores a batch on file and returns a stored batch. This method can be passed as *store* argument to create an open :class:`Sample` instance.
"""
#-------------------------------------------------------------------------------
with self.base.open('rb+') as u:
n = int(pickle.load(u))
u.seek(0)
pickle.dump(n + 1, u)
u.truncate()
p = self.getpath(n)
npsave(p, batch)
return npload(p, mmap_mode='r+')
def toSample(self):
r"""
Returns a closed :class:`Sample` instance.
"""
#-------------------------------------------------------------------------------
with self.base.open('rb') as u:
n = int(pickle.load(u))
L = [npload(self.getpath(k), mmap_mode='r') for k in range(n)]
sample = Sample(L[0].shape[1], None)
content = [(r, r_.shape[1]) for r, r_ in zip(L[:-1], L[1:])]
content.append((L[-1], 0))
sample.content = tuple(content)
sample.closed = True
return sample
def lith_data(self):
"""dictionary of data for lith volume"""
if getattr(self, '_lith_data', None) is None:
self._lith_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.vol.npy'):
continue
if npyfile.startswith('vol_'):
continue
self._lith_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False)).flatten()
return self._lith_data
def drill_data(self):
"""dictionry of drill point data"""
if getattr(self, '_drill_data', None) is None:
self._drill_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.point.npy'):
continue
if npyfile.endswith('_v.point.npy'):
continue
self._drill_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
return self._drill_data
def drill_data(self):
"""dictionry of drill point data"""
if getattr(self, '_drill_data', None) is None:
self._drill_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.point.npy'):
continue
if npyfile.endswith('_v.point.npy'):
continue
self._drill_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False)
)
return self._drill_data
def lith_data(self):
"""dictionary of data for lith volume"""
if getattr(self, '_lith_data', None) is None:
self._lith_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.vol.npy'):
continue
if npyfile.startswith('vol_'):
continue
self._lith_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False)
).flatten()
return self._lith_data
def xsect_data(self):
"""dictionary of cross section data"""
if getattr(self, '_xsect_data', None) is None:
self._xsect_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.xsurf.npy'):
continue
if (npyfile.endswith('_v.xsurf.npy')
or npyfile.endswith('_t.xsurf.npy')):
continue
self._xsect_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
return self._xsect_data
def borehole_data(self):
"""dictionary of borehole data"""
if getattr(self, '_borehole_data', None) is None:
self._borehole_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.line.npy'):
continue
if (npyfile.endswith('_v.line.npy')
or npyfile.endswith('_s.line.npy')):
continue
self._borehole_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
return self._borehole_data
def lith_diorite_early_data(self):
"""data for early diorite surface"""
return dict(
location='N',
data=DataArray(
title='dist_to_borehole',
array=npload(
Wolfpass.fetch_data(
filename='dist_to_borehole.lithsurf.npy',
download_if_missing=False,
verbose=False
)
)
)
)
def borehole_data(self):
"""dictionary of borehole data"""
if getattr(self, '_borehole_data', None) is None:
self._borehole_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.line.npy'):
continue
if (npyfile.endswith('_v.line.npy') or
npyfile.endswith('_s.line.npy')):
continue
self._borehole_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False)
)
return self._borehole_data
def xsect_data(self):
"""dictionary of cross section data"""
if getattr(self, '_xsect_data', None) is None:
self._xsect_data = dict()
for npyfile in self.filenames:
if not npyfile.endswith('.xsurf.npy'):
continue
if (npyfile.endswith('_v.xsurf.npy') or
npyfile.endswith('_t.xsurf.npy')):
continue
self._xsect_data[npyfile.split('.')[0]] = npload(
Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False)
)
return self._xsect_data
def borehole_raw_data(self):
"""list of title/array dictionaries for borehole data"""
raw_data = []
for npyfile in self.filenames:
if not npyfile.endswith('.line.npy'):
continue
if (npyfile.endswith('_v.line.npy') or
npyfile.endswith('_s.line.npy')):
continue
raw_data += [dict(
title=npyfile.split('.')[0],
array=npload(Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
)]
return raw_data
def lith_data(self):
"""raw data for lith volume"""
raw_data = []
for npyfile in self.filenames:
if not npyfile.endswith('.vol.npy'):
continue
if npyfile.startswith('vol_'):
continue
raw_data += [dict(
location='CC',
data=DataArray(
title=npyfile.split('.')[0],
array=npload(Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
)
)]
return raw_data
def __init__(self,fileName,evts,chans,det,simInfo, default=''):
self.fileName =fileName
self.det = det
self.evts = evts
self.chans = chans
self.tz, self.tStart, self.tStop = simInfo
self.trigT = self.tz
self.ebounds = npload("/Users/jburgess/Research/specSim/"+eboundDict[det]+'_ebounds.npy')
self.defaultRSP = default
self._CreatePrimaryHeader()
self._MakeEboundsEXT()
self._MakeEventsEXT()
self._MakeGTI()
self._MakeHDUList()
def xsect_data(self):
"""cross section raw data"""
raw_data = []
for npyfile in self.filenames:
if not npyfile.endswith('.xsurf.npy'):
continue
if (npyfile.endswith('_v.xsurf.npy') or
npyfile.endswith('_t.xsurf.npy')):
continue
raw_data += [dict(
location='CC',
data=DataArray(
title=npyfile.split('.')[0],
array=npload(Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
)
)]
return raw_data
def drill_data(self):
"""drill location raw data
list of dicts of 'location' and 'data' with 'title' and 'array'
"""
raw_data = []
for npyfile in self.filenames:
if not npyfile.endswith('.point.npy'):
continue
if npyfile.endswith('_v.point.npy'):
continue
raw_data += [dict(
location='N',
data=DataArray(
title=npyfile.split('.')[0],
array=npload(Wolfpass.fetch_data(filename=npyfile,
download_if_missing=False,
verbose=False))
)
)]
return raw_data
def xsect_triangles(self):
"""cross section triangles"""
return npload(Wolfpass.fetch_data(filename='xsect_t.xsurf.npy',
download_if_missing=False,
verbose=False))
def borehole_vertices(self):
"""borehole line vertices"""
return npload(
Wolfpass.fetch_data(filename='boreholes_v.line.npy',
download_if_missing=False,
verbose=False))
def topo_triangles(self):
"""topography triangles"""
return npload(Wolfpass.fetch_data(filename='topo_t.toposurf.npy',
download_if_missing=False,
verbose=False))
def load_trained_artefacts(filename='cartpole_artefacts.npy'):
saved_artefacts = npload(filename, allow_pickle=True)
return (saved_artefacts[()]['Q'],
saved_artefacts[()]['successful_episode_index'])
def lith_triangles(self):
"""list of lithology surface triangles"""
return [npload(Wolfpass.fetch_data(filename=prefix + '_t.lithsurf.npy',
download_if_missing=False,
verbose=False))
for prefix in self.lith_prefixes]
def lith_diorite_early_data(self):
"""data for early diorite surface"""
return npload(
Wolfpass.fetch_data(filename='dist_to_borehole.lithsurf.npy',
download_if_missing=False,
verbose=False))
print >> msgout, "Invalid answer: '%s'" % overwrite
overwrite = raw_input(
"Warning: Output file '%s' exists. Do you wish to overwrite file? (Y/N) "
% args.output)
if overwrite.upper() in ("Y", "YES"):
print >> msgout, "Overwriting '%s'." % args.output
print >> msgout, ""
elif overwrite.upper() in ("N", "NO"):
print >> msgout, "Operation aborted."
sys.exit()
else:
print >> sys.stderr, "Operation aborted. Cowardly refusing to overwrite '%s'." % args.output
sys.exit()
brt_type = npload(infile)
metadata = npload(infile)
t0 = time.time()
if args.input == "-":
print >> msgout, "Reading data from <stdin>...",
else:
print >> msgout, "Reading data from '%s'..." % args.input,
msgout.flush()
data = npload(infile)
if infile is not sys.stdin:
infile.close()
print >> msgout, "%.2f seconds" % (time.time() - t0)
if len(data.shape) != 2:
def __init__(self):
self.pathToDark = path.join("libs", "dark.npy")
self.darkData = npload(self.pathToDark)
self.darkScaled = self.darkData[:]
self.tLastScale = -1
self.scaleTimeOut = 60 # in seconds
reu_tf_idf_npz = load_npz(reu_tf_idf_npz) reu_ix_to_val = open(os.path.join(BASE, 'reuters/matrix_ix_to_val.json')) reu_ix_to_val = jsload(reu_ix_to_val) id_to_reu = open(os.path.join(BASE, 'reuters/id_to_reu_headline.csv')) id_to_reu = read_csv(id_to_reu) reu_vocab_to_ix = open(os.path.join(BASE, 'reuters/vocab_to_ix.json')) reu_vocab_to_ix = jsload(reu_vocab_to_ix) red_vocab_to_ix = open(os.path.join(BASE, 'reddit/vocab_to_ix.json')) red_vocab_to_ix = jsload(red_vocab_to_ix) words_compressed = open(os.path.join(BASE, 'u.npy'), 'rb') words_compressed = npload(words_compressed) docs_compressed = open(os.path.join(BASE, 'v_trans.npy'), 'rb') docs_compressed = npload(docs_compressed) ATN_word_to_ix = pload(open(os.path.join(BASE, 'allthenews_vocab_ix.p'))) reddit_ix_to_val = open(os.path.join(BASE, 'reddit/matrix_ix_to_val.json')) reddit_ix_to_val = jsload(reddit_ix_to_val) date_to_id = open(os.path.join(BASE, 'reuters/date_to_id.json')) date_to_id = jsload(date_to_id) red_text = pload(open(os.path.join(BASE, 'red_ix_to_text.p'), 'rb')) print >> sys.stderr, "LOADED FILES AT THE START" # searching_message = None
def topo_triangles(self):
"""topography triangles"""
return npload(
Wolfpass.fetch_data(filename='topo_t.toposurf.npy',
download_if_missing=False,
verbose=False))
def topo_data(self):
"""elevation data"""
return npload(
Wolfpass.fetch_data(filename='elevation.toposurf.npy',
download_if_missing=False,
verbose=False))
def lith_origin(self):
"""x0 for lith volume"""
return npload(
Wolfpass.fetch_data(filename='vol_x0.vol.npy',
download_if_missing=False,
verbose=False))
def lith_origin(self):
"""x0 for lith volume"""
return npload(Wolfpass.fetch_data(filename='vol_x0.vol.npy',
download_if_missing=False,
verbose=False))
def load_state(self, file_path):
self.holdkey_matrix = npload(file_path, allow_pickle=True).copy()
def drill_vertices(self):
"""drill point vertices"""
return npload(
Wolfpass.fetch_data(filename='drill_loc_v.point.npy',
download_if_missing=False,
verbose=False))
def xsect_triangles(self):
"""cross section triangles"""
return npload(
Wolfpass.fetch_data(filename='xsect_t.xsurf.npy',
download_if_missing=False,
verbose=False))
def drill_vertices(self):
"""drill point vertices"""
return npload(Wolfpass.fetch_data(filename='drill_loc_v.point.npy',
download_if_missing=False,
verbose=False))
def topo_data(self):
"""elevation data"""
return npload(Wolfpass.fetch_data(filename='elevation.toposurf.npy',
download_if_missing=False,
verbose=False))
dl = Data_Loading('./total_count.npy', './mfs_3342.csv')
dataset = dl.raw_data
genes_list = dl.genes_list
dataset = dataset.T
assert dataset.shape == (94655, 32738)
type_counts = [
10085, 2612, 9232, 10479, 11213, 8385, 11953, 10209, 10224, 10263
]
dn = DataNormalization(dataset, type_counts)
data_norm = dn.data
true_labels = []
for ti in range(10):
true_labels.extend([ti for i in range(type_counts[ti])])
repeats = 1
data = data_norm
from numpy import load as npload
total_genes = npload('./total_genes.npy')
for i in range(repeats):
for genes in genes_list[:]:
g_filter = [g in genes for g in total_genes]
data = data[:, g_filter]
#kmeans of different metrics
ce = Clustering_eval(data, true_labels, metrics='euclidean'
) # metrics: euclideanm,pearson or spearman
print('genes: %d,time used: %s, nmi: %f, ari: %f, fmi: %f' %
(len(genes.keys()), ce._time, ce.nmi, ce.ari, ce.fms))
data = data_norm # recover
def borehole_vertices(self):
"""borehole line vertices"""
return npload(Wolfpass.fetch_data(filename='boreholes_v.line.npy',
download_if_missing=False,
verbose=False))
print('===> Loading datasets')
# Load the training and testing kernels
Ktrain = th.load(opt.psfTrainPath)
for key in Ktrain.keys():
Ktrain[key] = Ktrain[key].float()
Ktest = th.load(opt.psfTestPath)
for key in Ktest.keys():
Ktest[key] = Ktest[key].float()
if opt.imdbPath == "":
Ntrain, Ntest = 400, 100
else:
f = npload(opt.imdbPath)
Ntrain = f['train_set'].shape[-1]
Ntest = f['test_set'].shape[-1]
del f
assert(opt.numTrainImagesperPSF <= Ntrain and opt.numTestImagesperPSF <= Ntest),\
"Invalid values for one or both of numTrainImagesperPSF and numTestImagesperPSF."
train_mask = getSubArrays(0,Ntrain,len(Ktrain),length=opt.numTrainImagesperPSF,dformat=lambda x:ndarray(x))
test_mask = getSubArrays(0,Ntest,len(Ktest),length=opt.numTestImagesperPSF,dformat=lambda x:ndarray(x))
train_data_loader = {}
test_data_loader = {}
for k in range(len(Ktrain)):
train_set = BSDS_deblur(Ktrain[k],opt.stdn,random_seed=opt.data_seed,filepath=opt.imdbPath,train=True,color=opt.color,shape=(256,256),batchSize=50,mask=train_mask[k])
train_data_loader[k] = DataLoader(dataset = train_set, num_workers=opt.threads, batch_size=opt.batchSize, shuffle=True)
for k in range(len(Ktest)):
test_set = BSDS_deblur(Ktest[k],opt.stdn,random_seed=opt.data_seed,filepath=opt.imdbPath,train=False,color=opt.color,shape=(256,256),batchSize=50,mask=test_mask[k])
def borehole_segments(self):
"""borehole segment vertex indices"""
return npload(Wolfpass.fetch_data(filename='boreholes_s.line.npy',
download_if_missing=False,
verbose=False))
def openModel(self, fDir, fName):
"""opens an orti file, stored in xml"""
self.fileDir, self.fileName = fDir, fName
self.initAll()
fullName = fDir + os.sep + fName
if fName + '.orti' in os.listdir(self.fileDir): fullName += '.orti'
else: fullName += '.iqpht'
flgArr = False
f1 = file(fullName, 'r')
doc = f1.read()
f1.close()
if 'compressdata.npz' in os.listdir(fDir):
darr = npload(fDir + os.sep + 'compressdata.npz')
flgArr = True
dom = xdom.parseString(doc)
dicts = dom.getElementsByTagName("dict")
for d in dicts:
dname = d.getElementsByTagName("name")[0].childNodes[0].data
model, typ = dname.split('_')
if (model not in self.modelList) & (model != 'dic'): continue
keys = d.getElementsByTagName("key")
dict1 = {}
for k in keys:
kname = str(
k.getElementsByTagName("name")[0].childNodes[0].data)
kdata = k.getElementsByTagName("content")[0].childNodes[0].data
exec('dict1[kname] = ' + kdata)
if typ == 'val':
self.dicval[model].update(dict1)
#print self.dicval[model]
elif typ == 'type':
self.dictype[model].update(dict1)
elif typ == 'zone':
self.diczone[model].setDic(dict1)
elif typ == 'array' and flgArr:
for k in keys:
kname = str(
k.getElementsByTagName("name")[0].childNodes[0].data)
self.dicarray[model][kname] = darr[kname.replace('.', '')]
elif typ == 'formula':
self.dicformula[model].update(dict1)
elif typ == 'addin':
self.addin.update1(dict1)
#print self.dicaddin
self.addin.grd = makeGrid(self, self.dicaddin['Grid'])
#print 'core 152',self.addin.grd
self.makeTtable()
self.Zblock = makeZblock(self)
mtype = self.dicaddin['Model']['group'][:5]
if mtype == 'Modfl':
self.flowReader = modflowReader(fDir, fName)
self.transReader = mtphtReader(fDir, fName)
elif mtype == 'Min3p':
self.flowReader = min3pReader(fDir, fName)
self.transReader = min3pReader(fDir, fName)
elif mtype == 'Opgeo':
self.addin.opgeo.buildMesh()
self.flowReader = ogReader(fDir, fName, 'flow')
self.transReader = ogReader(fDir, fName, 'trans')
elif mtype == 'Sutra':
self.flowReader = sutraReader(fDir, fName)
self.transReader = sutraReader(fDir, fName)
self.addin.setChemType()
def cu_triangles(self):
"""list of cu pct surface triangles"""
return [npload(Wolfpass.fetch_data(filename=prefix + '_t.cusurf.npy',
download_if_missing=False,
verbose=False))
for prefix in self.cu_prefixes]
"Warning: Output file '%s' exists. Do you wish to overwrite file? (Y/N) "
% args.output)
if overwrite.upper() in ("Y", "YES"):
print >> msgout, "Overwriting '%s'." % args.output
print >> msgout, ""
elif overwrite.upper() in ("N", "NO"):
print >> msgout, "Operation aborted."
sys.exit()
else:
print >> sys.stderr, "Operation aborted. Cowardly refusing to overwrite '%s'." % args.output
sys.exit()
outfile = args.output
tag = npload(infile)
metadata = npload(infile)
data = npload(infile)
if infile is not sys.stdin:
infile.close()
fdata = fft(data, axis=0)
if len(data.shape) != 2:
print >> sys.stderr, "Expected a two-dimensional array. Got an array with shape %s instead." % (
data.shape, )
sys.exit()
kind_str = {
"f": "floating point",
"i": "integer",
def borehole_segments(self):
"""borehole segment vertex indices"""
return npload(
Wolfpass.fetch_data(filename='boreholes_s.line.npy',
download_if_missing=False,
verbose=False))
