def perform_sample_manips(sampler, ndims, energy):
"""this function takes the mcmc sampler, grabs the chains from it, and
generates plots and percentiles and most likely values from the sampled
points"""
# retrieve the samples
num_samples = (CONFIG["Number of Walkers"] * (CONFIG["Sample Points"] -
CONFIG["Burn-in Points"]))
samples = sampler.chain[:, CONFIG["Burn-in Points"]:, :].reshape((
num_samples, ndims))
# save the samples to the disk in the sp
if CONFIG["Save Chain Data"]:
print "Saving MCMC samples for", energy, "MeV"
chain_file_name = ""
if CONFIG["Chain Directory"][-1] == '/':
chain_file_name = CONFIG["Chain Directory"] +\
"A%d_chain_E%4.1f.npz" % (CONFIG["Target A"], energy)
else:
chain_file_name = CONFIG["Chain Directory"] +\
"/A%d_chain_E%4.1f.npz" % (CONFIG["Target A"], energy)
np.savez_compressed(chain_file_name, sampler.chain)
print "Done saving MCMC samples for", energy, "MeV"
# extract the error bars
quantile_list = np.array([(0.5 - CONFIG["Confidence Interval"] / 2.0), 0.5,
(0.5 + CONFIG["Confidence Interval"] / 2.0)])
values = calc_param_values(samples, quantile_list, ndims)
peak_vals = [param[0] for param in values[1]]
# make the probability plots
make_prob_plots(samples, energy, peak_vals)
# make the corner plot
if CONFIG["Generate Corner Plots"]:
print "Commencing corner plot creation for", energy, "MeV"
lbls = [r"$a_{%d}$" % i for i in range(ndims)]
ranges = [(0.00, (0.0001 + samples[:, i].max())) for i in
range(ndims)]
fig = None
if CONFIG["Corner Plot Samples"] >= num_samples:
fig = corner.corner(samples, labels=lbls, range=ranges,
quantiles=quantile_list, truths=peak_vals,
verbose=False)
else:
# randomize the sample array and then extract the first chunk of it
np.random.shuffle(samples)
fig = corner.corner(samples[0:CONFIG["Corner Plot Samples"]],
labels=lbls, range=ranges,
quantiles=quantile_list, truths=peak_vals,
verbose=False)
# make the corner plot file_name
if CONFIG["Corner Plots Directory"][-1] == '/':
fig_file_name = CONFIG["Corner Plots Directory"] +\
"A%d_corner_E%4.1f.%s" % (CONFIG["Target A"], energy,
CONFIG["Plot Format"])
else:
fig_file_name = CONFIG["Corner Plots Directory"] +\
"/A%d_corner_E%4.1f.%s" % (CONFIG["Target A"], energy,
CONFIG["Plot Format"])
fig.savefig(fig_file_name, bbox_inches='tight')
plt.close(fig)
print "Done creating corner plot for", energy, "MeV"
# return the point and the errors
return values
def compute_descriptors(self):
descriptor_filename = '%s/descriptors.npz' % self.dataset_cache_dir
if os.path.exists(descriptor_filename):
self.descriptors = np.load(descriptor_filename)['descriptors']
else:
self.descriptors = self.captioner.compute_descriptors(self.images)
np.savez_compressed(descriptor_filename, descriptors=self.descriptors)
def compute_histogram(infiles, outfile):
"""
For each point, compute the distribution over pixel values.
"""
position_file = infiles[0]
basedir = infiles[1]
cf = pickle.load(open(infiles[2]))
start_f = cf['start_f']
end_f = cf['end_f']
frame_shape = cf['frame_dim_pix']
FRAMEN = end_f - start_f + 1
allframes = np.arange(FRAMEN)
def chunker(seq, size):
return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
# incremental variance calculation
hist = np.zeros((frame_shape[0], frame_shape[1], 256), dtype=np.float32)
for fis in chunker(allframes, 1000):
fs = get_frames(basedir, fis)
print fis[0]
for framei, frame in enumerate(fs):
cutil.frame_hist_add(hist, frame)
np.savez_compressed(outfile, hist=hist)
def save_data(self, fmt='npy', dest='.'):
if not self.has_saved:
#must specify a unique new directory name for this output
if not os.path.exists(dest):
os.makedirs(dest)
sim_dir = os.path.join(dest,self.fname)
i = 1
while os.path.exists(sim_dir):
sim_dir = os.path.join(dest,self.fname,'-%i'%i)
i += 1
os.mkdir(sim_dir)
self.sim_dir = sim_dir
self.has_saved = True
fname = os.path.join(self.sim_dir, self.fname)
params = {k:self.__dict__[k] for k in self.__dict__ if k not in ['cells','neuropil','t','mov','mov_nofilter','stim','mov_nojit','mov_filtered']}
cells = [cell.get_dict() for cell in self.cells]
npil = np.array([self.neuropil.get_dict()])
t = self.__dict__['t']
stim = self.__dict__['stim']
movie = np.array([{k:self.__dict__[k] for k in ['mov','mov_nofilter','mov_nojit','mov_filtered']}])
if fmt in ['npy','npz','numpy','n']:
np.savez_compressed(fname, params=params, cells=cells, neuropil=npil, time=t, stim=stim, movie=movie)
elif fmt in ['mat','matlab','m']:
matdic = {'params':params, 'cells':cells, 'neuropil':npil, 'time':t, 'stim':stim, 'movie':movie}
savemat(fname, matdic)
def save_images_to_disk():
print('Disk-saving thread active...')
n = 0
frameTimeOutputFile = open(planOutputPath+'frameTimes.txt','w')
frameTimeOutputFile.write('frameCount\t n\t frameCond\t frameT\t interval\n')
currdict = im_queue.get()
while currdict is not None:
frameTimeOutputFile.write('%i\t %i\t %i\t %s\t %s\n' % (int(currdict['frame']),n,int(currdict['cond']),currdict['time'],currdict['interval']))
if save_as_tiff:
fname = '%s/frame%i.tiff' % (dataOutputPath,int(currdict['frame']))
tiff = TIFF.open(fname, mode='w')
tiff.write_image(currdict['im'])
tiff.close()
elif save_as_npz:
np.savez_compressed('%s/test%d.npz' % (output_path, n), currdict['im'])
else:
fname = '%s/frame%i.tiff' % (dataOutputPath,int(currdict['frame']),)
with open(fname, 'wb') as f:
pkl.dump(currdict, f, protocol=pkl.HIGHEST_PROTOCOL)
# print 'DONE SAVING FRAME: ', currdict['frame'], n #fdict
n += 1
currdict = im_queue.get()
disk_writer_alive = False
#frameTimeOutputFile.close()
print('Disk-saving thread inactive...')
def parse_to_word_vectors(input_file, output_dir, wordvectors, window, total_sentences, debug):
print('Loading vectors', file=sys.stderr)
if not debug:
word2vec_model = gensim.models.Word2Vec.load_word2vec_format(wordvectors, binary=True)
else:
word2vec_model = gensim.models.Word2Vec()
instance_extractor = WordVectorsExtractor(word2vec_model, window)
instances = []
words = []
print('Getting instances from corpus {}'.format(input_file), file=sys.stderr)
parser = WikipediaCorpusColumnParser(input_file)
for sentence in tqdm(parser, total=total_sentences):
sentence_words = [(word.idx, word.token, word.tag, word.is_doc_start)
for word in sentence]
sentence_instances, _, _ = instance_extractor.get_instances_for_sentence(sentence, 0)
assert len(sentence_words) == len(sentence_instances)
instances.extend(sentence_instances)
words.extend(sentence_words)
print('Saving matrix and words', file=sys.stderr)
dataset_matrix = np.vstack(instances)
np.savez_compressed(os.path.join(output_dir, 'evaluation_dataset_word_vectors.npz'), dataset=dataset_matrix)
with open(os.path.join(output_dir, 'evaluation_words_word_vectors.pickle'), 'wb') as f:
pickle.dump(words, f)
def save_npz(file, obj, compression=True):
"""Saves an object to the file in NPZ format.
This is a short-cut function to save only one object into an NPZ file.
Args:
file (str or file-like): Target file to write to.
obj: Object to be serialized. It must support serialization protocol.
compression (bool): If ``True``, compression in the resulting zip file
is enabled.
.. seealso::
:func:`chainer.serializers.load_npz`
"""
if isinstance(file, six.string_types):
with open(file, 'wb') as f:
save_npz(f, obj, compression)
return
s = DictionarySerializer()
s.save(obj)
if compression:
numpy.savez_compressed(file, **s.target)
else:
numpy.savez(file, **s.target)
def dump(self, path):
"""Save the log to an npz file.
Stores the log in structured form in an npz file. The resulting file can be
extracted using unzip, which will write every leaf node to its own file in
an equivalent directory structure.
Args:
path: the path of the npz file to which to save.
"""
dikt = {}
def _compile_npz_dict(item, path):
i, node = item
if isinstance(node, _Section):
for subitem in node.items:
_compile_npz_dict(subitem,
os.path.join(path, "%s_%s" % (i, node.label)))
else:
for k, v in node.items():
dikt[os.path.join(path, "%s_%s" % (i, k))] = v
_compile_npz_dict((0, self.root), "")
with lib_util.atomic_file(path) as p:
np.savez_compressed(p, **dikt)
def main():
if len(sys.argv) < 2:
print("Not enough arguments supplied")
return
datapath = sys.argv[1]
train_image_path = os.path.join(datapath, "train_images.npz")
test_image_path = os.path.join(datapath, "test_images.npz")
train_cap_path = os.path.join(datapath, "train_labels.txt")
test_cap_path = os.path.join(datapath, "test_labels.txt")
trX, teX, trY, teY = mnist(onehot=False)
print("Generating Appended MNIST Training...")
train_imgs, train_caps = generate_dataset(trX, trY)
print("Generating Appended MNIST Testing...")
test_imgs, test_caps = generate_dataset(teX, teY)
print("Save Training/Testing Images...")
np.savez_compressed(train_image_path, *train_imgs)
np.savez_compressed(test_image_path, *test_imgs)
print("Save Training/Testing Captions...")
with open(train_cap_path, 'w') as train_cap_file:
train_cap_file.writelines(train_caps)
with open(test_cap_path, 'w') as test_cap_file:
test_cap_file.writelines(test_caps)
print("DONE. SUMMARY")
print("# Train Examples: " + str(len(train_imgs)))
print("# Test Examples: " + str(len(test_imgs)))
def do_save(grid, arrays):
from tvtk.api import tvtk
from tvtk.api import write_data
pd = tvtk.PolyData()
pd.points = grid.leaf_view.vertices.T
pd.polys = grid.leaf_view.elements
pd.point_data.scalars = grid.leaf_view.domain_indices
pd.point_data.scalars.name = "domains"
write_data(pd, "test_kitchen_sink_grid.vtk")
abn = {a.type:a.data for a in arrays}
mgrid = abn["mgrid"]
potential = abn["gscalar"]
gradient = abn["gvector"]
print dimensions, potential.shape
print 'spacing:',spacing
print 'origin:',origin
print 'dimensions:',dimensions
sp = tvtk.StructuredPoints(spacing=spacing, origin=origin, dimensions=dimensions)
sp.point_data.scalars = potential.ravel(order='F')
sp.point_data.scalars.name = "potential"
sp.point_data.vectors = gradient.ravel(order='F')
sp.point_data.vectors.name = "gradient"
write_data(sp, "test_kitchen_sink_potential.vtk")
numpy.savez_compressed("test_kitchen_sink.npz", **abn)
def do_gen(content, decision):
SER_S = np.empty((tau_range.shape[0], phi_range.shape[0]))
SER_U = np.empty((tau_range.shape[0], phi_range.shape[0]))
for tau_idx, tau in enumerate(tau_range):
print("tau = %.2f" % (tau))
# symbols of (synch"ed, unsynch"ed) sender
if content in ("same",):
tmp_syms = np.random.randint(16, size=nsyms + 2)
send_syms = [tmp_syms, tmp_syms]
else:
send_syms = np.random.randint(16, size=2 * (nsyms + 2)).reshape(2, nsyms + 2)
send_syms_s, send_syms_u = send_syms[0][1:-1], send_syms[1][1:-1]
send_chips = pt.map_chips(*send_syms)
RECV_CHIPS_I = pt.detect_i(send_chips[:2], send_chips[2:], phi_range, tau, As, Au)
RECV_CHIPS_Q = pt.detect_q(send_chips[:2], send_chips[2:], phi_range, tau, As, Au)
for phi_idx in range(len(phi_range)):
recv_chips = np.empty(2 * RECV_CHIPS_I.shape[0])
recv_chips[::2] = RECV_CHIPS_I[:, phi_idx]
recv_chips[1::2] = RECV_CHIPS_Q[:, phi_idx]
# slice bits to simulate hard decision decoder
if decision in ("hard",):
recv_chips = np.sign(recv_chips)
recv_syms = pt.detect_syms_corr(recv_chips)[1:-1]
SER_S[tau_idx, phi_idx] = sum(recv_syms != send_syms_s) / (1.0 * len(recv_syms))
SER_U[tau_idx, phi_idx] = sum(recv_syms != send_syms_u) / (1.0 * len(recv_syms))
np.savez_compressed("data/ser_Au%.2f_%s_%s_v2.npz" % (Au, content, decision), SER_S=SER_S, SER_U=SER_U, **settings)
def _download_higgs_data_and_save_npz(data_dir):
"""Download higgs data and store as a numpy compressed file."""
input_url = os.path.join(URL_ROOT, INPUT_FILE)
np_filename = os.path.join(data_dir, NPZ_FILE)
if tf.gfile.Exists(np_filename):
raise ValueError('data_dir already has the processed data file: {}'.format(
np_filename))
if not tf.gfile.Exists(data_dir):
tf.gfile.MkDir(data_dir)
# 2.8 GB to download.
try:
print('Data downloading..')
temp_filename, _ = urllib.request.urlretrieve(input_url)
# Reading and parsing 11 million csv lines takes 2~3 minutes.
print('Data processing.. taking multiple minutes..')
data = pd.read_csv(
temp_filename,
dtype=np.float32,
names=['c%02d' % i for i in range(29)] # label + 28 features.
).as_matrix()
finally:
os.remove(temp_filename)
# Writing to temporary location then copy to the data_dir (0.8 GB).
f = tempfile.NamedTemporaryFile()
np.savez_compressed(f, data=data)
tf.gfile.Copy(f.name, np_filename)
print('Data saved to: {}'.format(np_filename))
def save_cb(self, mode):
"""Save image, figure, and plot data arrays."""
# This just defines the basename.
# Extension has to be explicitly defined or things can get messy.
target = Widgets.SaveDialog(
title='Save {0} data'.format(mode)).get_path()
# Save cancelled
if not target:
return
# TODO: This can be a user preference?
fig_dpi = 100
if mode == 'cuts':
# Save as fits file
image = self.fitsimage.get_image()
self.fv.error_wrap(image.save_as_file, target + '.fits')
fig, xarr, yarr = self.cuts_plot.get_data()
elif mode == 'slit':
fig, xarr, yarr = self.slit_plot.get_data()
fig.savefig(target + '.png', dpi=fig_dpi)
numpy.savez_compressed(target + '.npz', x=xarr, y=yarr)
def _save_results(self, save_layers, sess):
# Train loss
np.savetxt(os.path.join(self.results_save_path, 'train_loss_record_%s.txt' % self.experiment_name),
np.array(self.train_loss_record, dtype=np.float32), fmt='%.3f', delimiter=',')
# Validation accuracy
np.savetxt(os.path.join(self.results_save_path, 'validation_accuracy_record_%s.txt' % self.experiment_name),
np.array(self.validation_accuracy_record, dtype=np.float32), fmt='%.3f', delimiter=',')
# Test
self.test_results.to_csv(os.path.join(self.results_save_path, 'test_results_%s.csv' % self.experiment_name),
index=False)
self.test_predictions_results.to_csv(
os.path.join(self.results_save_path, 'test_predictions_%s.csv' % self.experiment_name), index=False)
if save_layers:
print('Saving weights and biases', file=sys.stderr)
file_name_weights = os.path.join(self.pre_trained_weights_save_path,
"%s_weights.npz" % self.experiment_name)
file_name_biases = os.path.join(self.pre_trained_weights_save_path,
"%s_biases.npz" % self.experiment_name)
weights_dict = {}
biases_dict = {}
for layer_idx, (weights, biases) in enumerate(zip(self.weights, self.biases)):
layer_name = 'hidden_layer_%02d' % layer_idx
weights_dict[layer_name] = weights.eval(session=sess)
biases_dict[layer_name] = biases.eval(session=sess)
np.savez_compressed(file_name_weights, **weights_dict)
np.savez_compressed(file_name_biases, **biases_dict)
def main():
if os.path.exists('points.npz'):
print("Loading points from points.npz")
points = np.load('points.npz')['points']
else:
points = get_points()
print("Saving points to points.npz")
np.savez_compressed('points.npz', points=points)
def point_input():
for x, y in points:
yield '%s %s\n' % (x, y)
b_i = []
b_j = []
b_x = []
b_y = []
t1 = time.time()
for line in run_subprocess(('./union',), point_input()):
i, j, x, y = line.split()
b_i.append(int(i))
b_j.append(int(j))
b_x.append(float(x))
b_y.append(float(y))
t2 = time.time()
b_i = np.asarray(b_i)
b_j = np.asarray(b_j)
b_x = np.asarray(b_x)
b_y = np.asarray(b_y)
print("Got %d boundary intersections" % len(b_i) +
" in %.4f s" % (t2 - t1))
np.savez_compressed('boundary.npz', i=b_i, j=b_j, x=b_x, y=b_y)
def save_cb(self, mode):
"""Save image, figure, and plot data arrays."""
# This just defines the basename.
# Extension has to be explicitly defined or things can get messy.
w = Widgets.SaveDialog(title='Save {0} data'.format(mode))
filename = w.get_path()
if filename is None:
# user canceled dialog
return
# TODO: This can be a user preference?
fig_dpi = 100
if mode == 'cuts':
fig, xarr, yarr = self.cuts_plot.get_data()
elif mode == 'slit':
fig, xarr, yarr = self.slit_plot.get_data()
figname = filename + '.png'
self.logger.info("saving figure as: %s" % (figname))
fig.savefig(figname, dpi=fig_dpi)
dataname = filename + '.npz'
self.logger.info("saving data as: %s" % (dataname))
np.savez_compressed(dataname, x=xarr, y=yarr)
def _storeresult(result):
"""Test function to try various ways to store a tuple of numpy arrays.
"""
if False:
# Try numpy
npy.savez_compressed('store-npy.npz', *result)
if False:
# Try h5py
import h5py
store = h5py.File("store-h5py.hdf5", "w", compression='lzf')
store['numoccs'] = numoccs
store['occcounts'] = occcounts
store['childoccfreqs'] = childoccfreqs
store['numunique'] = numunique
store['uniquecounts'] = childuniquefreqs
store['childuniquefreqs'] = childuniquefreqs
if False:
# Try PyTables
import tables
store = tables.open_file(
'store-pytables.hdf5', mode="w",
filters=tables.Filters(complib='bzip2', complevel=6))
def storearray(name, x):
atom = tables.Atom.from_dtype(x.dtype)
ds = store.createCArray(store.root, name, atom, x.shape)
ds[:] = x
storearray('numoccs', numoccs)
storearray('occcounts', occcounts)
storearray('childoccfreqs', childoccfreqs)
storearray('numunique', numunique)
storearray('uniquecounts', childuniquefreqs)
storearray('childuniquefreqs', childuniquefreqs)
store.close()
def align(movie_data, options, args, lrh):
print 'pPCA(scikit-learn)'
nvoxel = movie_data.shape[0]
nTR = movie_data.shape[1]
nsubjs = movie_data.shape[2]
align_algo = args.align_algo
nfeature = args.nfeature
# zscore the data
bX = np.nan((nsubjs*nvoxel,nTR))
for m in xrange(nsubjs):
bX[m*nvoxel:(m+1)*nvoxel,:] = stats.zscore(movie_data[:, :, m].T, axis=0, ddof=1).T
del movie_data
U, s, VT = np.linalg.svd(bX, full_matrices=False)
bW = np.zeros((nsubjs*nvoxel,nfeature))
for m in xrange(nsubjs):
bW[m*nvoxel:(m+1)*nvoxel,:] = U[m*nvoxel:(m+1)*nvoxel,:nfeature]
niter = 10
# initialization when first time run the algorithm
np.savez_compressed(options['working_path']+align_algo+'_'+lrh+'_'+str(niter)+'.npz',\
bW = bW, niter=niter)
return niter
def save_pruned_depthmap(self, image, points, normals, colors, labels, detections):
io.mkdir_p(self._depthmap_path())
filepath = self._depthmap_file(image, 'pruned.npz')
np.savez_compressed(filepath,
points=points, normals=normals,
colors=colors, labels=labels,
detections=detections)
def testFileSeekableWithZip(self):
# Note: Test case for GitHub issue 27276, issue only exposed in python 3.7+.
filename = os.path.join(self._base_dir, "a.npz")
np.savez_compressed(filename, {"a": 1, "b": 2})
with gfile.GFile(filename, "rb") as f:
info = np.load(f, allow_pickle=True)
_ = [i for i in info.items()]
def save(filename, grid, dirichlet_fun, neumann_fun):
lv = grid.leaf_view
import numpy
numpy.savez_compressed(filename,
elements=lv.elements, vertices=lv.vertices, domains=lv.domain_indices,
dirichlet_coefficients = dirichlet_fun.coefficients,
neumann_coefficients = neumann_fun.coefficients)
def write_npz_file(self):
assert len(self.examples) >= self.Nperfile
# put Nperfile random examples at the end of the list
for i in xrange(self.Nperfile):
a = len(self.examples) - i - 1
if a > 0:
b = random.randint(0, a-1)
self.examples[a], self.examples[b] = self.examples[b], self.examples[a]
# pop Nperfile examples off the end of the list
# put each component into a separate numpy batch array
save_dict = {}
for c in xrange(len(self.names)):
batch_shape = (self.Nperfile,) + self.shapes[c]
batch = np.empty(batch_shape, dtype=self.dtypes[c])
for i in xrange(self.Nperfile):
batch[i,:] = self.examples[-1-i][c]
save_dict[self.names[c]] = batch
del self.examples[-self.Nperfile:]
filename = os.path.join(self.out_dir, "examples.%d.%d" % (self.Nperfile, self.filenum))
#print "NPZ.RandomizingWriter: writing", filename
np.savez_compressed(filename, **save_dict)
self.filenum += 1
def creator(path):
archive_path = download.cached_download(url)
train_x = numpy.empty((5, 10000, 3072), dtype=numpy.uint8)
train_y = numpy.empty((5, 10000), dtype=numpy.uint8)
test_y = numpy.empty(10000, dtype=numpy.uint8)
dir_name = '{}-batches-py'.format(name)
with tarfile.open(archive_path, 'r:gz') as archive:
# training set
for i in range(5):
file_name = '{}/data_batch_{}'.format(dir_name, i + 1)
d = pickle.load(archive.extractfile(file_name))
train_x[i] = d['data']
train_y[i] = d['labels']
# test set
file_name = '{}/test_batch'.format(dir_name)
d = pickle.load(archive.extractfile(file_name))
test_x = d['data']
test_y[...] = d['labels'] # copy to array
train_x = train_x.reshape(50000, 3072)
train_y = train_y.reshape(50000)
numpy.savez_compressed(path, train_x=train_x, train_y=train_y,
test_x=test_x, test_y=test_y)
return {'train_x': train_x, 'train_y': train_y,
'test_x': test_x, 'test_y': test_y}
def split_dataset(labels, indices_save_path, classes_save_path, train_size=0.8,
test_size=0.1, validation_size=0.1, min_count=3):
classes = {}
print('Getting YAGO labels', file=sys.stderr, flush=True)
yago_labels = [label[1] for label in labels]
print('Getting filtered classes', file=sys.stderr, flush=True)
filtered_classes = {l for l, v in Counter(yago_labels).items() if v >= min_count}
print('Getting filtered indices', file=sys.stderr, flush=True)
filtered_indices = np.array([i for i, l in enumerate(yago_labels)
if (l != 'O' and l in filtered_classes) or (l == 'O')], dtype=np.int32)
strat_split = StratifiedSplitter(np.array(yago_labels), filtered_indices)
print('Splitting the dataset', file=sys.stderr, flush=True)
train_indices, test_indices, validation_indices = strat_split.get_splitted_dataset_indices(
train_size=train_size, test_size=test_size, validation_size=validation_size)
print('Saving indices to file %s' % indices_save_path, file=sys.stderr, flush=True)
np.savez_compressed(indices_save_path, train_indices=train_indices, test_indices=test_indices,
validation_indices=validation_indices, filtered_indices=filtered_indices)
for idx, iteration in enumerate(CL_ITERATIONS[::-1]):
print('Getting classes for iteration %s' % iteration, file=sys.stderr, flush=True)
replaced_labels = [label[idx] for label in labels]
classes[iteration] = np.unique(np.array(replaced_labels)[filtered_indices], return_counts=True)
print('Saving classes to file %s' % classes_save_path, file=sys.stderr, flush=True)
with open(classes_save_path, 'wb') as f:
pickle.dump(classes, f)
def feature_selection(dataset, features_names, matrix_file_path, features_file_path, max_features=12000):
print('Calculating variance of dataset features', file=sys.stderr, flush=True)
dataset = csc_matrix(dataset)
square_dataset = dataset.copy()
square_dataset.data **= 2
variance = np.asarray(square_dataset.mean(axis=0) - np.square(dataset.mean(axis=0)))[0]
print('Getting top %d features' % max_features, file=sys.stderr, flush=True)
top_features = np.argsort(variance)[::-1][:max_features]
min_variance = variance[top_features][-1]
print('Min variance: %.2e. Getting features over min variance.' % min_variance, file=sys.stderr, flush=True)
valid_indices = np.where(variance > min_variance)[0]
print('Final features count: %d/%d' % (valid_indices.shape[0], dataset.shape[1]), file=sys.stderr, flush=True)
print('Filtering features', file=sys.stderr, flush=True)
dataset = csr_matrix(dataset[:, valid_indices])
print('Saving dataset to file {}'.format(matrix_file_path), file=sys.stderr, flush=True)
np.savez_compressed(matrix_file_path, data=dataset.data, indices=dataset.indices,
indptr=dataset.indptr, shape=dataset.shape)
print('Saving filtered features names', file=sys.stderr, flush=True)
features_names = np.array(features_names)
filtered_features_names = list(features_names[valid_indices])
with open(features_file_path, 'wb') as f:
pickle.dump(filtered_features_names, f)
def align(movie_data, options, args, lrh):
print 'PCA'
nvoxel = movie_data.shape[0]
nTR = movie_data.shape[1]
nsubjs = movie_data.shape[2]
align_algo = args.align_algo
nfeature = args.nfeature
if not os.path.exists(options['working_path']):
os.makedirs(options['working_path'])
# zscore the data
bX = np.zeros((nsubjs*nvoxel,nTR))
for m in range(nsubjs):
bX[m*nvoxel:(m+1)*nvoxel,:] = stats.zscore(movie_data[:,:,m].T ,axis=0, ddof=1).T
del movie_data
U, s, VT = np.linalg.svd(bX, full_matrices=False)
bW = U[:,range(nfeature)]
ES = np.diag(s).dot(VT)
ES = ES[:nfeature,:]
niter = 10
# initialization when first time run the algorithm
np.savez_compressed(options['working_path']+align_algo+'_'+lrh+'_'+str(niter)+'.npz',\
bW = bW, ES=ES, niter=niter)
return niter
def process_glove(args, vocab_list, save_path, size=4e5, random_init=True):
"""
:param vocab_list: [vocab]
:return:
"""
if not gfile.Exists(save_path + ".npz"):
glove_path = os.path.join(args.glove_dir, "glove.6B.{}d.txt".format(args.glove_dim))
if random_init:
glove = np.random.randn(len(vocab_list), args.glove_dim)
else:
glove = np.zeros((len(vocab_list), args.glove_dim))
found = 0
with open(glove_path, 'r') as fh:
for line in tqdm(fh, total=size):
array = line.lstrip().rstrip().split(" ")
word = array[0]
vector = list(map(float, array[1:]))
if word in vocab_list:
idx = vocab_list.index(word)
glove[idx, :] = vector
found += 1
if word.capitalize() in vocab_list:
idx = vocab_list.index(word.capitalize())
glove[idx, :] = vector
found += 1
if word.upper() in vocab_list:
idx = vocab_list.index(word.upper())
glove[idx, :] = vector
found += 1
print("{}/{} of word vocab have corresponding vectors in {}".format(found, len(vocab_list), glove_path))
np.savez_compressed(save_path, glove=glove)
print("saved trimmed glove matrix at: {}".format(save_path))
def save_vocab(self, path_count, path_vocab, word_limit=100000):
""" Saves the master vocabulary into a file.
"""
# reserve space for 10 special tokens
words = OrderedDict()
for token in SPECIAL_TOKENS:
# store -1 instead of np.inf, which can overflow
words[token] = -1
# sort words by frequency
desc_order = OrderedDict(sorted(self.master_vocab.items(),
key=lambda kv: kv[1], reverse=True))
words.update(desc_order)
# use encoding of up to 30 characters (no token conversions)
# use float to store large numbers (we don't care about precision loss)
np_vocab = np.array(words.items(),
dtype=([('word','|S30'),('count','float')]))
# output count for debugging
counts = np_vocab[:word_limit]
np.savez_compressed(path_count, counts=counts)
# output the index of each word for easy lookup
final_words = OrderedDict()
for i, w in enumerate(words.keys()[:word_limit]):
final_words.update({w:i})
with open(path_vocab, 'w') as f:
f.write(json.dumps(final_words, indent=4, separators=(',', ': ')))
def main():
desc = 'Applies summarize_scores on a given score matrix'
parser = argparse.ArgumentParser(description = desc)
parser.add_argument('infile')
parser.add_argument('idfile', help = 'File with region ids')
parser.add_argument('outfile')
args = parser.parse_args()
data = np.load(args.infile)
scores = data['scores']
motif_names = data['motif_names']
data.close()
is_score = np.array([not re.search('_scores', m) is None for m in motif_names], np.bool)
is_count = np.array([not s for s in is_score], np.bool)
region_ids = []
with open(args.idfile, 'r') as infile:
for line in infile:
if line.strip() == '.':
region_ids.append(None)
else:
region_ids.append(line.strip())
scores_tmp, new_names = summarize_scores(scores[:, is_score], region_ids, np.max)
counts_tmp, new_names_tmp = summarize_scores(scores[:, is_count], region_ids, np.sum)
motif_names = np.array(motif_names)
motif_names = list(np.concatenate((motif_names[is_score], motif_names[is_count])))
scores = np.concatenate((scores_tmp, counts_tmp), axis = 1)
np.savez_compressed(args.outfile, scores = scores, motif_names = motif_names,
region_names = new_names)
def write_sparse_matrix(matrix, filepath, compressed=True):
"""
Write a ``scipy.sparse.csr_matrix`` or ``scipy.sparse.csc_matrix`` to disk
at ``filepath``, optionally compressed.
Args:
matrix (``scipy.sparse.csr_matrix`` or ``scipy.sparse.csr_matrix``)
filepath (str): /path/to/file on disk to which matrix objects will be written;
if ``filepath`` does not end in ``.npz``, that extension is
automatically appended to the name
compressed (bool): if True, save arrays into a single file in compressed
.npz format
.. seealso: http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.savez.html
.. seealso: http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.savez_compressed.html
"""
if not isinstance(matrix, (csc_matrix, csr_matrix)):
raise TypeError('input matrix must be a scipy sparse csr or csc matrix')
make_dirs(filepath, 'w')
if compressed is False:
savez(filepath,
data=matrix.data, indices=matrix.indices,
indptr=matrix.indptr, shape=matrix.shape)
else:
savez_compressed(filepath,
data=matrix.data, indices=matrix.indices,
indptr=matrix.indptr, shape=matrix.shape)
# Get facial image
face = extract_face(model, pixels)
if face is None:
continue
faces.append(face)
if len(faces) >= n:
break
return np.asarray(faces)
def plot_faces(faces, n):
for i in range(n*n):
plt.subplot(n, n, i+1)
plt.axis('off')
plt.imshow(faces[i])
plt.savefig("artifacts/sample_faces.png")
if __name__ == "__main__":
dirname = "data/img_align_celeba"
# faces = load_faces(dirname, 25)
# print("Loaded faces shape: ", faces.shape)
# plot_faces(faces, 5)
n = 50_000
all_faces = load_faces(dirname, n)
print("Loaded: ", all_faces.shape)
np.savez_compressed("img_align_celeba.npz", all_faces)
def save_weights(self):
weights = {
name: p.get_value()
for name, p in LasagneNetwork._get_named_params(self.out_layer)
}
np.savez_compressed(self.filepath, **weights)
def train_med2vec(seqFile='seqFile.txt',
demoFile='demoFile.txt',
labelFile='labelFile.txt',
outFile='outFile.txt',
modelFile='modelFile.txt',
L2_reg=0.001,
numXcodes=20000,
numYcodes=20000,
embDimSize=1000,
hiddenDimSize=2000,
batchSize=100,
demoSize=2,
logEps=1e-8,
windowSize=1,
verbose=False,
maxEpochs=1000):
options = locals().copy()
print 'initializing parameters'
params = init_params(options)
#params = load_params(options)
tparams = init_tparams(params)
print 'building models'
f_grad_shared = None
f_update = None
if demoSize > 0 and numYcodes > 0:
x, d, y, mask, iVector, jVector, cost = build_model(tparams, options)
grads = T.grad(cost, wrt=tparams.values())
f_grad_shared, f_update = adadelta(tparams,
grads,
x,
mask,
iVector,
jVector,
cost,
options,
d=d,
y=y)
elif demoSize == 0 and numYcodes > 0:
x, y, mask, iVector, jVector, cost = build_model(tparams, options)
grads = T.grad(cost, wrt=tparams.values())
f_grad_shared, f_update = adadelta(tparams,
grads,
x,
mask,
iVector,
jVector,
cost,
options,
y=y)
elif demoSize > 0 and numYcodes == 0:
x, d, mask, iVector, jVector, cost = build_model(tparams, options)
grads = T.grad(cost, wrt=tparams.values())
f_grad_shared, f_update = adadelta(tparams,
grads,
x,
mask,
iVector,
jVector,
cost,
options,
d=d)
else:
x, mask, iVector, jVector, cost = build_model(tparams, options)
grads = T.grad(cost, wrt=tparams.values())
f_grad_shared, f_update = adadelta(tparams, grads, x, mask, iVector,
jVector, cost, options)
print 'loading data'
seqs, demos, labels = load_data(seqFile, demoFile, labelFile)
n_batches = int(np.ceil(float(len(seqs)) / float(batchSize)))
print 'training start'
for epoch in xrange(maxEpochs):
iteration = 0
costVector = []
for index in random.sample(range(n_batches), n_batches):
batchX = seqs[batchSize * index:batchSize * (index + 1)]
batchY = []
batchD = []
if demoSize > 0 and numYcodes > 0:
batchY = labels[batchSize * index:batchSize * (index + 1)]
x, y, mask, iVector, jVector = padMatrix(
batchX, batchY, options)
batchD = demos[batchSize * index:batchSize * (index + 1)]
cost = f_grad_shared(x, batchD, y, mask, iVector, jVector)
elif demoSize == 0 and numYcodes > 0:
batchY = labels[batchSize * index:batchSize * (index + 1)]
x, y, mask, iVector, jVector = padMatrix(
batchX, batchY, options)
cost = f_grad_shared(x, y, mask, iVector, jVector)
elif demoSize > 0 and numYcodes == 0:
x, mask, iVector, jVector = padMatrix(batchX, batchY, options)
batchD = demos[batchSize * index:batchSize * (index + 1)]
cost = f_grad_shared(x, batchD, mask, iVector, jVector)
else:
x, mask, iVector, jVector = padMatrix(batchX, batchY, options)
cost = f_grad_shared(x, mask, iVector, jVector)
costVector.append(cost)
f_update()
if (iteration % 10 == 0) and verbose:
print 'epoch:%d, iteration:%d/%d, cost:%f' % (epoch, iteration,
n_batches, cost)
iteration += 1
print 'epoch:%d, mean_cost:%f' % (epoch, np.mean(costVector))
tempParams = unzip(tparams)
np.savez_compressed(outFile + '.' + str(epoch), **tempParams)
# axs[i].set_ylim([0.9*ymin[i]/vthE, 1.1*ymax[i]/vthE])
# axs[i].set_xlabel("$x/\lambda_D$")
# axs[i].set_xlim([0,lx_norm])
# axs[i].set_title(labels[i]+" (time = %f"%time+" $t\omega_{pi}$)")
xI = np.array(xI)
vI = np.array(vI)
xE = np.array(xE)
vE = np.array(vE)
time = np.array(time)
# print(xI.shape,vI.shape,time.shape)
timer.task('Save data')
np.savez_compressed(pjoin(folder, 'xv_data.npz'),
time=time,
xI=xI,
vI=vI,
xE=xE,
vE=vE)
##### FIG SIZE CALC ############
figsize = np.array([77, 77 * 1.618]) #Figure size in mm
dpi = 300 #Print resolution
ppi = np.sqrt(1920**2 + 1200**2) / 24 #Screen resolution
mp.rc('text', usetex=True)
mp.rc('font', family='sans-serif', size=10, serif='Computer Modern Roman')
mp.rc('axes', titlesize=10)
mp.rc('axes', labelsize=10)
mp.rc('xtick', labelsize=10)
mp.rc('ytick', labelsize=10)
mp.rc('legend', fontsize=10)
elif plane == 'Z':
pred__[:, :, minR] /= 2
pred__[:, :, minR + 1:maxR - 1] /= 3
pred__[:, :, maxR - 1] /= 2
print(' Testing is finished: ' +
str(time.time() - start_time) +
' second(s) elapsed.')
pred_ = pred_ + pred__
pred[pred_ > 3 / 2] = 1
#pred = post_processing(pred, pred, 0.5, organ_ID)
print(' Testing is finished: ' + \
str(time.time() - start_time) + ' second(s) elapsed.')
np.savez_compressed(volume_file, volume=pred)
else:
volume_data = np.load(volume_file)
pred = volume_data['volume']
print(' Testing result is loaded: ' + \
str(time.time() - start_time) + ' second(s) elapsed.')
DSC[r,
i], inter_sum, pred_sum, label_sum = DSC_computation(
label, pred)
print(' DSC = 2 * ' + str(inter_sum) + ' / (' + str(pred_sum) + ' + ' + \
str(label_sum) + ') = ' + str(DSC[r, i]) + ' .')
output = open(result_file, 'a+')
output.write(' Round ' + str(r) + ', ' + 'DSC = 2 * ' + str(inter_sum) + ' / (' + \
str(pred_sum) + ' + ' + str(label_sum) + ') = ' + str(DSC[r, i]) + ' .\n')
output.close()
for i in range(len(features)):
feature_array[i,:len(features[i]),:] = features[i][:max_seq_length]
feature_array[i,len(features[i]):,:] = padding_feature
ss3_labels = ['H', 'E', 'C', 'NA']
ss8_labels = ['H', 'G', 'I', 'E', 'B', 'T', 'S', 'L', 'NA']
ss3_labels_lookup = {'H': ['G','H','I'], 'E':['E','B'], 'C':['S','T','L', 'NA']}
ss3_labels_lookup_inv = {'H':'H', 'G':'H', 'I':'H', 'E':'E', 'B':'E', 'T':'C', 'S':'C', 'L':'C'}
ss_output_dim = len(ss3_labels)
dummy_indices = [3]
padding_label = np.zeros(ss_output_dim)
padding_label[dummy_indices] = 1
label_array = np.zeros([len(labels), max_seq_length, ss_output_dim])
for i in range(len(labels)):
label_ss8_indices = labels[i]
label_ss3_indices = [ss3_labels.index(ss3_labels_lookup_inv[ss8_labels[label_index]]) for label_index in label_ss8_indices]
# print label_ss3_indices
# print label_array[i]
label_array[i,:len(labels[i])] = np.eye(ss_output_dim)[label_ss3_indices]
label_array[i,len(labels[i]):] = padding_label
# label_array[i,:,label_ss3_indices] = 1
data = np.concatenate([feature_array, label_array], axis=2)
np.savez_compressed(options.output_data_filename, data=data)
print(data.shape)
else:
tracks2, d02 = loadData(str(dataSet))
tracks = np.concatenate((tracks, tracks2))
d0 = np.concatenate((d0, d02))
print("Total Tracks: " + str(tracks.shape))
indices = np.arange(len(tracks))
np.random.shuffle(indices)
tracks = tracks[indices]
d0 = d0[indices]
model = callModel()
model.load_weights(weightsDir + 'model.99.h5')
predictions = model.predict(tracks)
pred_fakes = np.argwhere(predictions >= 0.5)
pred_reals = np.argwhere(predictions < 0.5)
print("Number of predicted fakes: " + str(len(pred_fakes)) + ", Number of predicted Reals: " + str(len(pred_reals)))
#plotMetrics.predictionCorrelation(predictions, d0, 20, -1, 1, 'predictionD0Correlation', plotDir)
#plotMetrics.comparePredictions(predictions, d0, 20, -1, 1, 'd0', plotDir)
plotMetrics.plotScores(predictions, sys.argv[1], plotDir)
np.savez_compressed(outputDir + "predictions.npz", tracks = tracks, d0 = d0, predictions = predictions)
print "SEED: ", traj_seed
err = init_instability(inst, traj_seed)
if err == 1:
print 'Bad trajectory! ', ii
inst.set_pert_seed(ii)
inst.run_dynamics()
answers.append(inst.distance)
lambdas.append(inst.lambdas[0])
lambdas_no_regr.append(inst.lambdas_no_regr[0])
polarisation.append(inst.polarisation)
polarisation1.append(inst.polarisation1)
dist.append(inst.distance)
energy.append(inst.energy)
rhosq = (inst.X ** 2 + inst.Y ** 2)
rho1sq = (inst.X1 ** 2 + inst.Y1 ** 2)
rev_idx = np.arange(rho1sq.shape[3])[::-1]
full_tmp.append(np.sqrt(np.sum((rhosq - rho1sq[:,:,:,rev_idx]) ** 2, axis=(0,1,2))))
np.savez_compressed(vis.filename(my_id),
full=full_tmp,
data=answers, lambdas=lambdas, lambdas_no_regr=lambdas_no_regr,
polar=polarisation, polar1=polarisation1,
distance=dist, energy=energy,
step=inst.step, time=inst.time, n_steps=inst.n_steps,
well_indices=inst.wells_indices,
my_info=[seed_from, seed_to, my_id],
needed_trajs=needed_trajs,
description='Loschmidt echo for 10000 replicates, trying to calculate F(t)')
def main(
video_path: str,
exp_cfg,
show: bool = False,
demo_output_folder: str = 'demo_output',
pause: float = -1,
focal_length: float = 5000,
rcnn_batch: int = 1,
sensor_width: float = 36,
save_vis: bool = True,
save_params: bool = False,
save_mesh: bool = False,
degrees: Optional[List[float]] = [],
) -> None:
device = torch.device('cuda')
if not torch.cuda.is_available():
logger.error('CUDA is not available!')
sys.exit(3)
logger.remove()
logger.add(lambda x: tqdm.write(x, end=''),
level=exp_cfg.logger_level.upper(),
colorize=True)
# 画像フォルダ作成
image_folder = osp.join(osp.dirname(osp.abspath(video_path)),
osp.basename(video_path).replace(".", "_"))
os.makedirs(image_folder, exist_ok=True)
# 動画を静画に変えて出力
idx = 0
cap = cv2.VideoCapture(video_path)
# 幅と高さを取得
original_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
original_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
while (cap.isOpened()):
# 動画から1枚キャプチャして読み込む
flag, frame = cap.read() # Capture frame-by-frame
# 終わったフレームより後は飛ばす
# 明示的に終わりが指定されている場合、その時も終了する
if flag == False:
break
cv2.imwrite(osp.join(image_folder, "capture_{0:012d}.png".format(idx)),
frame)
idx += 1
cap.release()
expose_dloader = preprocess_images(image_folder + "/*.png",
exp_cfg,
batch_size=rcnn_batch,
device=device)
demo_output_folder = osp.join(
osp.expanduser(osp.expandvars(demo_output_folder)),
osp.basename(video_path).replace(".", "_"),
datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
logger.info(f'Saving results to: {demo_output_folder}')
os.makedirs(demo_output_folder, exist_ok=True)
#関節位置情報ファイル
posf = open(osp.join(demo_output_folder, 'pos.txt'), 'w')
model = SMPLXNet(exp_cfg)
try:
model = model.to(device=device)
except RuntimeError:
# Re-submit in case of a device error
sys.exit(3)
output_folder = exp_cfg.output_folder
checkpoint_folder = osp.join(output_folder, exp_cfg.checkpoint_folder)
checkpointer = Checkpointer(model,
save_dir=checkpoint_folder,
pretrained=exp_cfg.pretrained)
arguments = {'iteration': 0, 'epoch_number': 0}
extra_checkpoint_data = checkpointer.load_checkpoint()
for key in arguments:
if key in extra_checkpoint_data:
arguments[key] = extra_checkpoint_data[key]
model = model.eval()
means = np.array(exp_cfg.datasets.body.transforms.mean)
std = np.array(exp_cfg.datasets.body.transforms.std)
render = save_vis or show
body_crop_size = exp_cfg.get('datasets', {}).get('body',
{}).get('transforms').get(
'crop_size', 256)
if render:
hd_renderer = HDRenderer(img_size=body_crop_size)
total_time = 0
cnt = 0
for bidx, batch in enumerate(tqdm(expose_dloader, dynamic_ncols=True)):
full_imgs_list, body_imgs, body_targets = batch
if full_imgs_list is None:
continue
full_imgs = to_image_list(full_imgs_list)
body_imgs = body_imgs.to(device=device)
body_targets = [target.to(device) for target in body_targets]
full_imgs = full_imgs.to(device=device)
torch.cuda.synchronize()
start = time.perf_counter()
model_output = model(body_imgs,
body_targets,
full_imgs=full_imgs,
device=device)
torch.cuda.synchronize()
elapsed = time.perf_counter() - start
cnt += 1
total_time += elapsed
hd_imgs = full_imgs.images.detach().cpu().numpy().squeeze()
body_imgs = body_imgs.detach().cpu().numpy()
body_output = model_output.get('body')
_, _, H, W = full_imgs.shape
# logger.info(f'{H}, {W}')
# H, W, _ = hd_imgs.shape
if render:
hd_imgs = np.transpose(undo_img_normalization(hd_imgs, means, std),
[0, 2, 3, 1])
hd_imgs = np.clip(hd_imgs, 0, 1.0)
right_hand_crops = body_output.get('right_hand_crops')
left_hand_crops = torch.flip(body_output.get('left_hand_crops'),
dims=[-1])
head_crops = body_output.get('head_crops')
bg_imgs = undo_img_normalization(body_imgs, means, std)
right_hand_crops = undo_img_normalization(right_hand_crops, means,
std)
left_hand_crops = undo_img_normalization(left_hand_crops, means,
std)
head_crops = undo_img_normalization(head_crops, means, std)
body_output = model_output.get('body', {})
num_stages = body_output.get('num_stages', 3)
stage_n_out = body_output.get(f'stage_{num_stages - 1:02d}', {})
model_vertices = stage_n_out.get('vertices', None)
if stage_n_out is not None:
model_vertices = stage_n_out.get('vertices', None)
faces = stage_n_out['faces']
if model_vertices is not None:
model_vertices = model_vertices.detach().cpu().numpy()
camera_parameters = body_output.get('camera_parameters', {})
camera_scale = camera_parameters['scale'].detach()
camera_transl = camera_parameters['translation'].detach()
out_img = OrderedDict()
final_model_vertices = None
stage_n_out = model_output.get('body', {}).get('final', {})
if stage_n_out is not None:
final_model_vertices = stage_n_out.get('vertices', None)
if final_model_vertices is not None:
final_model_vertices = final_model_vertices.detach().cpu().numpy()
camera_parameters = model_output.get('body', {}).get(
'camera_parameters', {})
camera_scale = camera_parameters['scale'].detach()
camera_transl = camera_parameters['translation'].detach()
hd_params = weak_persp_to_blender(
body_targets,
camera_scale=camera_scale,
camera_transl=camera_transl,
H=H,
W=W,
sensor_width=sensor_width,
focal_length=focal_length,
)
if save_vis:
bg_hd_imgs = np.transpose(hd_imgs, [0, 3, 1, 2])
out_img['hd_imgs'] = bg_hd_imgs
if render:
# Render the initial predictions on the original image resolution
hd_orig_overlays = hd_renderer(
model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
)
out_img['hd_orig_overlay'] = hd_orig_overlays
# Render the overlays of the final prediction
if render:
# bbox保持
cbbox = body_targets[0].bbox.detach().cpu().numpy()
bbox_size = np.array(body_targets[0].size)
dset_center = np.array(body_targets[0].extra_fields['center'])
dset_size = np.array(body_targets[0].extra_fields['bbox_size'])
# 画面サイズに合わせる(描画のため、int)
bbox = np.tile(dset_center, 2) + (
(cbbox / np.tile(bbox_size, 2) - np.tile(0.5, 4)) *
np.tile(dset_size, 4))
img_bbox = bbox.astype(np.int)
hd_params['img_bbox'] = bbox
hd_overlays = hd_renderer(
final_model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
body_color=[0.4, 0.4, 0.7])
proj_joints = stage_n_out['proj_joints'][0].detach().cpu().numpy()
hd_params['proj_joints'] = proj_joints
try:
# 横線
for x in range(img_bbox[0], img_bbox[2] + 1):
for y in [img_bbox[1], img_bbox[3] + 1]:
if hd_overlays.shape[2] > x and hd_overlays[3] > y:
hd_overlays[:, :, y, x] = np.array([1, 0, 0, 1])
# 縦線
for x in [img_bbox[0], img_bbox[2] + 1]:
for y in range(img_bbox[1], img_bbox[3] + 1):
if hd_overlays.shape[2] > x and hd_overlays[3] > y:
hd_overlays[:, :, y, x] = np.array([1, 0, 0, 1])
# カメラ中央
for x in range(int(hd_params['center'][0, 0] - 1),
int(hd_params['center'][0, 0] + 2)):
for y in range(int(hd_params['center'][0, 1] - 1),
int(hd_params['center'][0, 1] + 2)):
if hd_overlays.shape[2] > x and hd_overlays[3] > y:
hd_overlays[:, :, y, x] = np.array([0, 1, 0, 1])
min_joints = np.min(proj_joints, axis=0)
max_joints = np.max(proj_joints, axis=0)
diff_joints = max_joints - min_joints
diff_bbox = np.array([
hd_params['img_bbox'][2] - hd_params['img_bbox'][0],
hd_params['img_bbox'][3] - hd_params['img_bbox'][1]
])
jscale = diff_joints / diff_bbox
jscale = np.mean([jscale[0], jscale[1]])
for jidx, jname in enumerate(KEYPOINT_NAMES):
j2d = proj_joints[jidx] / jscale
# ジョイント
for x in range(int(hd_params['center'][0, 0] + j2d[0] - 1),
int(hd_params['center'][0, 0] + j2d[0] +
2)):
for y in range(
int(hd_params['center'][0, 1] + j2d[1] - 1),
int(hd_params['center'][0, 1] + j2d[1] + 2)):
if hd_overlays.shape[2] > x and hd_overlays[3] > y:
hd_overlays[:, :, y,
x] = np.array([0, 0, 1, 1])
except Exception as e:
print('hd_overlays error: %s' % e)
pass
out_img['hd_overlay'] = hd_overlays
for deg in degrees:
hd_overlays = hd_renderer(
final_model_vertices,
faces,
focal_length=hd_params['focal_length_in_px'],
camera_translation=hd_params['transl'],
camera_center=hd_params['center'],
bg_imgs=bg_hd_imgs,
return_with_alpha=True,
render_bg=False,
body_color=[0.4, 0.4, 0.7],
deg=deg,
)
out_img[f'hd_rendering_{deg:03.0f}'] = hd_overlays
if save_vis:
for key in out_img.keys():
out_img[key] = np.clip(
np.transpose(out_img[key], [0, 2, 3, 1]) * 255, 0,
255).astype(np.uint8)
for idx in tqdm(range(len(body_targets)), 'Saving ...'):
# TODO 複数人対応
if idx > 0:
break
fname = body_targets[idx].get_field('fname')
curr_out_path = osp.join(demo_output_folder, fname)
os.makedirs(curr_out_path, exist_ok=True)
if save_vis:
for name, curr_img in out_img.items():
pil_img.fromarray(curr_img[idx]).save(
osp.join(curr_out_path, f'{name}.png'))
if save_mesh:
# Store the mesh predicted by the body-crop network
naive_mesh = o3d.geometry.TriangleMesh()
naive_mesh.vertices = Vec3d(model_vertices[idx] +
hd_params['transl'][idx])
naive_mesh.triangles = Vec3i(faces)
mesh_fname = osp.join(curr_out_path, f'body_{fname}.ply')
o3d.io.write_triangle_mesh(mesh_fname, naive_mesh)
# Store the final mesh
expose_mesh = o3d.geometry.TriangleMesh()
expose_mesh.vertices = Vec3d(final_model_vertices[idx] +
hd_params['transl'][idx])
expose_mesh.triangles = Vec3i(faces)
mesh_fname = osp.join(curr_out_path, f'{fname}.ply')
o3d.io.write_triangle_mesh(mesh_fname, expose_mesh)
if save_params:
params_fname = osp.join(curr_out_path, f'{fname}_params.npz')
out_params = dict(fname=fname)
for key, val in stage_n_out.items():
if torch.is_tensor(val):
val = val.detach().cpu().numpy()[idx]
out_params[key] = val
for key, val in hd_params.items():
if torch.is_tensor(val):
val = val.detach().cpu().numpy()
if np.isscalar(val[idx]):
out_params[key] = val[idx].item()
else:
out_params[key] = val[idx]
try:
for param_name in ['center']:
params_txt_fname = osp.join(
curr_out_path, f'{fname}_params_{param_name}.txt')
np.savetxt(params_txt_fname, out_params[param_name])
for param_name in ['img_bbox']:
params_txt_fname = osp.join(
curr_out_path, f'{fname}_params_{param_name}.txt')
np.savetxt(params_txt_fname, hd_params[param_name])
for param_name in ['joints']:
params_txt_fname = osp.join(
curr_out_path, f'{fname}_params_{param_name}.json')
# json出力
joint_dict = {}
joint_dict["image"] = {"width": W, "height": H}
joint_dict["depth"] = {
"depth": float(hd_params["depth"][0][0])
}
joint_dict["center"] = {
"x": float(hd_params['center'][0, 0]),
"y": float(hd_params['center'][0, 1])
}
joint_dict["bbox"] = {
"x": float(hd_params["img_bbox"][0]),
"y": float(hd_params["img_bbox"][1]),
"width": float(hd_params["img_bbox"][2]),
"height": float(hd_params["img_bbox"][3])
}
joint_dict["joints"] = {}
joint_dict["proj_joints"] = {}
proj_joints = hd_params["proj_joints"]
joints = out_params["joints"]
min_joints = np.min(proj_joints, axis=0)
max_joints = np.max(proj_joints, axis=0)
diff_joints = max_joints - min_joints
diff_bbox = np.array([
hd_params['img_bbox'][2] -
hd_params['img_bbox'][0],
hd_params['img_bbox'][3] - hd_params['img_bbox'][1]
])
jscale = diff_joints / diff_bbox
jscale = np.mean([jscale[0], jscale[1]])
for jidx, jname in enumerate(KEYPOINT_NAMES):
j2d = proj_joints[jidx] / jscale
joint_dict["proj_joints"][jname] = {
'x': float(hd_params['center'][0, 0] + j2d[0]),
'y': float(hd_params['center'][0, 1] + j2d[1])
}
joint_dict["joints"][jname] = {
'x': float(joints[jidx][0]),
'y': float(-joints[jidx][1]),
'z': float(joints[jidx][2])
}
with open(params_txt_fname, 'w') as f:
json.dump(joint_dict, f, indent=4)
# 描画設定
fig = plt.figure(figsize=(15, 15), dpi=100)
# 3DAxesを追加
ax = fig.add_subplot(111, projection='3d')
# ジョイント出力
ax.set_xlim3d(int(-(original_width / 2)),
int(original_width / 2))
ax.set_ylim3d(0, int(original_height / 2))
ax.set_zlim3d(0, int(original_height))
ax.set(xlabel='x', ylabel='y', zlabel='z')
xs = []
ys = []
zs = []
for j3d_from_idx, j3d_to_idx in ALL_CONNECTIONS:
jfname = KEYPOINT_NAMES[j3d_from_idx]
jtname = KEYPOINT_NAMES[j3d_to_idx]
xs = [
joint_dict[jfname]['x'],
joint_dict[jtname]['x']
]
ys = [
joint_dict[jfname]['y'],
joint_dict[jtname]['y']
]
zs = [
joint_dict[jfname]['z'],
joint_dict[jtname]['z']
]
ax.plot3D(xs,
ys,
zs,
marker="o",
ms=2,
c="#0000FF")
plt.savefig(
os.path.join(curr_out_path,
f'{fname}_{param_name}.png'))
plt.close()
# posの出力
joint_names = [(0, 'pelvis'), (1, 'right_hip'), (2, 'right_knee'), (3, 'right_ankle'), \
(6, 'left_hip'), (7, 'left_knee'), (8, 'left_ankle'), \
(12, 'spine1'), (13, 'spine2'), (14, 'neck'), (15, 'head'), \
(17, 'left_shoulder'), (18, 'left_elbow'), (19, 'left_wrist'), \
(25, 'right_shoulder'), (26, 'right_elbow'), (27, 'right_wrist')
]
N = []
I = []
for (jnidx, iname) in joint_names:
for jidx, smplx_jn in enumerate(KEYPOINT_NAMES):
if smplx_jn == iname:
N.append(jnidx)
I.append([
joint_dict[iname]['x'],
joint_dict[iname]['y'],
joint_dict[iname]['z']
])
for i in np.arange(len(I)):
# 0: index, 1: x軸, 2:Y軸, 3:Z軸
posf.write(
str(N[i]) + " " + str(I[i][0]) + " " +
str(I[i][2]) + " " + str(I[i][1]) + ", ")
#終わったら改行
posf.write("\n")
except Exception as e:
print('savetxt error: %s' % e)
pass
np.savez_compressed(params_fname, **out_params)
if show:
nrows = 1
ncols = 4 + len(degrees)
fig, axes = plt.subplots(ncols=ncols,
nrows=nrows,
num=0,
gridspec_kw={
'wspace': 0,
'hspace': 0
})
axes = axes.reshape(nrows, ncols)
for ax in axes.flatten():
ax.clear()
ax.set_axis_off()
axes[0, 0].imshow(hd_imgs[idx])
axes[0, 1].imshow(out_img['rgb'][idx])
axes[0, 2].imshow(out_img['hd_orig_overlay'][idx])
axes[0, 3].imshow(out_img['hd_overlay'][idx])
start = 4
for deg in degrees:
axes[0, start].imshow(
out_img[f'hd_rendering_{deg:03.0f}'][idx])
start += 1
plt.draw()
if pause > 0:
plt.pause(pause)
else:
plt.show()
fmt = cv2.VideoWriter_fourcc(*'mp4v')
# メッシュの結合
writer = cv2.VideoWriter(osp.join(demo_output_folder, 'hd_overlay.mp4'),
fmt, 30, (original_width, original_height))
for img_path in glob.glob(osp.join(demo_output_folder,
"**/hd_overlay.png")):
writer.write(cv2.imread(img_path))
writer.release()
# JOINTの結合
writer = cv2.VideoWriter(osp.join(demo_output_folder, 'joints.mp4'), fmt,
30, (1500, 1500))
for img_path in glob.glob(osp.join(demo_output_folder, "**/*_joints.png")):
writer.write(cv2.imread(img_path))
writer.release()
cv2.destroyAllWindows()
posf.close()
logger.info(f'Average inference time: {total_time / cnt}')
# N(theta_new, theta^(t-1)_j, sigma)
pdf_theta = mvn.pdf(theta_new,
mean=data_calculation[j, 0:6],
cov=np.diag(sigma))
sum_w += w_j * pdf_theta
w = mvn.pdf(theta_new, mean=mu_prior, cov=sigma_prior) / sum_w
# new samples
data_calculation[i, 0:6] = theta_new
data_calculation[i, 6] = w
data_calculation[i, 7] = pho
if pho < kesi:
p_acc_cal += 1
p_acc = p_acc_cal / (N - N * alpha)
print(p_acc)
index = np.argsort(data_calculation[:, -1])
data_calculation = data_calculation[index]
kesi = data_calculation[int(N * alpha), -1]
std = np.std(data_calculation[0:int(N * alpha), 0:6], 0)
print('Iter' + str(t) + '_std:', std)
sigma = 2 * np.var(data_calculation[0:int(N * alpha), 0:6], 0)
P_acc_record.append(p_acc)
Kesi_record.append(kesi)
Sigma_record.append(sigma)
Appro_poster.append(data_calculation)
save_name = 'ABC_results'
np.savez_compressed(save_name, a=Appro_poster, b=Kesi_record)
def save_segmentation_nifti_from_softmax(segmentation_softmax,
out_fname,
dct,
order=1,
region_class_order=None,
seg_postprogess_fn=None,
seg_postprocess_args=None,
resampled_npz_fname=None,
non_postprocessed_fname=None,
force_separate_z=None):
"""
This is a utility for writing segmentations to nifto and npz. It requires the data to have been preprocessed by
GenericPreprocessor because it depends on the property dictionary output (dct) to know the geometry of the original
data. segmentation_softmax does not have to have the same size in pixels as the original data, it will be
resampled to match that. This is generally useful because the spacings our networks operate on are most of the time
not the native spacings of the image data.
If seg_postprogess_fn is not None then seg_postprogess_fnseg_postprogess_fn(segmentation, *seg_postprocess_args)
will be called before nifto export
There is a problem with python process communication that prevents us from communicating obejcts
larger than 2 GB between processes (basically when the length of the pickle string that will be sent is
communicated by the multiprocessing.Pipe object then the placeholder (\%i I think) does not allow for long
enough strings (lol). This could be fixed by changing i to l (for long) but that would require manually
patching system python code.) We circumvent that problem here by saving softmax_pred to a npy file that will
then be read (and finally deleted) by the Process. save_segmentation_nifti_from_softmax can take either
filename or np.ndarray for segmentation_softmax and will handle this automatically
:param segmentation_softmax:
:param out_fname:
:param dct:
:param order:
:param region_class_order:
:param seg_postprogess_fn:
:param seg_postprocess_args:
:param resampled_npz_fname:
:param non_postprocessed_fname:
:param force_separate_z: if None then we dynamically decide how to resample along z, if True/False then always
/never resample along z separately. Do not touch unless you know what you are doing
:return:
"""
if isinstance(segmentation_softmax, str):
assert isfile(segmentation_softmax), "If isinstance(segmentation_softmax, str) then " \
"isfile(segmentation_softmax) must be True"
del_file = deepcopy(segmentation_softmax)
segmentation_softmax = np.load(segmentation_softmax)
os.remove(del_file)
# first resample, then put result into bbox of cropping, then save
current_shape = segmentation_softmax.shape
shape_original_after_cropping = dct.get('size_after_cropping')
shape_original_before_cropping = dct.get('original_size_of_raw_data')
# current_spacing = dct.get('spacing_after_resampling')
# original_spacing = dct.get('original_spacing')
if np.any(
np.array(current_shape) != np.array(shape_original_after_cropping)
):
if force_separate_z is None:
if get_do_separate_z(dct.get('original_spacing')):
do_separate_z = True
lowres_axis = get_lowres_axis(dct.get('original_spacing'))
elif get_do_separate_z(dct.get('spacing_after_resampling')):
do_separate_z = True
lowres_axis = get_lowres_axis(
dct.get('spacing_after_resampling'))
else:
do_separate_z = False
lowres_axis = None
else:
do_separate_z = force_separate_z
if do_separate_z:
lowres_axis = get_lowres_axis(dct.get('original_spacing'))
else:
lowres_axis = None
print("separate z:", do_separate_z, "lowres axis", lowres_axis)
seg_old_spacing = resample_data_or_seg(segmentation_softmax,
shape_original_after_cropping,
is_seg=False,
axis=lowres_axis,
order=order,
do_separate_z=do_separate_z,
cval=0)
#seg_old_spacing = resize_softmax_output(segmentation_softmax, shape_original_after_cropping, order=order)
else:
seg_old_spacing = segmentation_softmax
if resampled_npz_fname is not None:
np.savez_compressed(resampled_npz_fname,
softmax=seg_old_spacing.astype(np.float16))
save_pickle(dct, resampled_npz_fname[:-4] + ".pkl")
if region_class_order is None:
seg_old_spacing = seg_old_spacing.argmax(0)
else:
seg_old_spacing_final = np.zeros(seg_old_spacing.shape[1:])
for i, c in enumerate(region_class_order):
seg_old_spacing_final[seg_old_spacing[i] > 0.5] = c
seg_old_spacing = seg_old_spacing_final
bbox = dct.get('crop_bbox')
if bbox is not None:
seg_old_size = np.zeros(shape_original_before_cropping)
for c in range(3):
bbox[c][1] = np.min((bbox[c][0] + seg_old_spacing.shape[c],
shape_original_before_cropping[c]))
seg_old_size[bbox[0][0]:bbox[0][1], bbox[1][0]:bbox[1][1],
bbox[2][0]:bbox[2][1]] = seg_old_spacing
else:
seg_old_size = seg_old_spacing
if seg_postprogess_fn is not None:
seg_old_size_postprocessed = seg_postprogess_fn(
np.copy(seg_old_size), *seg_postprocess_args)
else:
seg_old_size_postprocessed = seg_old_size
seg_resized_itk = sitk.GetImageFromArray(
seg_old_size_postprocessed.astype(np.uint8))
seg_resized_itk.SetSpacing(dct['itk_spacing'])
seg_resized_itk.SetOrigin(dct['itk_origin'])
seg_resized_itk.SetDirection(dct['itk_direction'])
sitk.WriteImage(seg_resized_itk, out_fname)
if (non_postprocessed_fname is not None) and (seg_postprogess_fn
is not None):
seg_resized_itk = sitk.GetImageFromArray(seg_old_size.astype(np.uint8))
seg_resized_itk.SetSpacing(dct['itk_spacing'])
seg_resized_itk.SetOrigin(dct['itk_origin'])
seg_resized_itk.SetDirection(dct['itk_direction'])
sitk.WriteImage(seg_resized_itk, non_postprocessed_fname)
def fit_interval(self, tmin=None, tmax=None, plot=None, save=None):
sl = fu.get_window_slice(tmin * us, self.td, tmax * us)
V = self.Vps[sl]
td = self.td[sl]
dt = self.dt
# Vline=np.array([]) #to save fitted line
# tline=np.array([]) #to save time point for fitted line (even though they should be same as td)
if len(V) > 1000000:
print('Too much to plot, will skip plotting.')
plot = False
plot_s = False #do not plot if interval is not specified
Vstep = self.par['Vstep']
Vth = self.par['Vth']
sig = self.par['sig']
if sig == None:
dtmin = self.par['dtmin']
n_samp = self.par['n_samp']
ts = td[0:n_samp] - dtmin
sig = fu.peak(ts, 1 / self.par['decay_time'],
1 / self.par['rise_time'])[0]
bkg_len = self.var['bkg_len']
vary_codes_bkg = self.var['vary_codes_bkg']
poly_order = self.par['poly_order']
def lish(x): #''' Vectorize the line shape'''
return LF.line_shape(x)
line_shape = np.vectorize(lish)
alpha = B.Parameter(1. / self.par['decay_time'], 'alpha')
beta = B.Parameter(1. / self.par['rise_time'], 'beta')
results = np.zeros((2, ), dtype='int32')
pmin = np.zeros((len(V) / 5, ), dtype='int32')
pmax = np.zeros((len(V) / 5, ), dtype='int32')
FP.find_peaks(len(V), Vstep, V, results, pmin, pmax)
## number of maxima
n_max = results[1]
print(" found : ", n_max, " maxima")
imax = pmax[:n_max]
## number of minima
nmin = results[0]
print(" found : ", nmin, " minima")
imin = pmin[:nmin]
## get the indices of peaks higher then threshold
ipeak_tr = np.delete(np.where(V[imax] > Vth)[0], -1)
#choose minimums close to previous maximum (to filter out electrical switching noise peaks)
closete = np.where((td[imin][ipeak_tr] - td[imax][ipeak_tr]) < 0.5)[0]
# check if next minimum is not too negative (removing switching noize)
clles = np.where(V[imin][ipeak_tr][closete] < -0.3)[0]
ipeak = np.delete(ipeak_tr, closete[clles])
# peak locations after electrical switching noise filtering
Vp = V[imax][ipeak]
tp = td[imax][ipeak]
imax_fit = imax[ipeak] #indeces of peaks to fit in raw data array
# loop over the peaks
Np = Vp.shape[0]
t_start = time.clock()
# forming fit groups
n_sig_low = self.par['n_sig_low']
n_sig_boundary = self.par['n_sig_boundary']
n_peaks_to_fit = self.par['n_peaks_to_fit']
#--------------------------------
# define window edges for fitting
#--------------------------------
# lower edge
dt_l = n_sig_low * sig
dl = int(dt_l / dt)
boundary = n_sig_boundary * sig
in_boundary = (np.zeros_like(tp) == 1)
# determine the fit groups
fg, fw = fu.get_fit_groups_new(n_peaks_to_fit, imax_fit, 0., td)
fg_shift, fw = fu.get_fit_groups_new(n_peaks_to_fit, imax_fit, fw / 2., td)
#if interval is specified for fitting
if tmin and tmax:
if (tmin * us >= self.par['dtmin']) and (
tmax * us <= self.par['dtmax']) and tmin < tmax:
#find fit groups covering the interval
# print td[imax_fit][0], td[imax_fit][-1], tmin*us, tmax*us
inmin = np.where(td[imax_fit] >= tmin * us)[0][
0] #index of first peak in fitting interval in time data
in_max = np.where(
td[imax_fit] <= tmax * us)[0][-1] #index of tmax in time data
gtf = np.empty((0, 2), int) #list of groups in interval to fit
gtfs = np.empty((0, 2), int)
for f in fg:
if f[0] > in_max and f[1] > in_max:
break
if f[0] > inmin or f[1] > inmin:
gtf = np.vstack([gtf, f])
for f in fg_shift:
if f[0] > in_max and f[1] > in_max:
break
if f[0] > inmin or f[1] > inmin:
gtfs = np.vstack([gtfs, f])
#plot=True
#pl.figure()
fg = gtf
fg_shift = gtfs #to fit only specified groups
else:
print("Interval out of range or incorrect.",
self.par['dtmin'] / us, self.par['dtmax'] / us)
return
# the number of peaks in each fit group
fg_n_peaks = fg[:, 1] - fg[:, 0] + 1
fg_shift_n_peaks = fg_shift[:, 1] - fg_shift[:, 0] + 1
# loop over fit groups and define peaks in the boundary area
# arrays for storing fit results
A_fit = np.zeros_like(tp)
sig_A_fit = np.zeros_like(tp)
# bkg fit parameters
bkg_par = np.zeros(shape=(len(tp), bkg_len))
#--------------------------------
#--------------------------------
# loop over fit groups for fitting
N_fitted = 0
lims = []
ifailed = 0
if plot:
pl.xlabel('t(us)')
pl.ylabel('V')
pl.title('Not shifted fitting groups')
for i, ff in enumerate(fg[:]):
N_fitted += fg_n_peaks[i]
if (i % 10 == 0) & (i != 0):
t_current = time.clock()
a_rate = 0.
t_diff = (t_current - t_start)
if (t_diff != 0.):
a_rate = float(N_fitted) / t_diff
print("Fit ", i,
float(N_fitted) / Np * 100., "% completed, time ", t_current,
' rate =', a_rate)
# form a slice for the current peaks
sl = slice(ff[0], ff[1] + 1)
# times for the peaks
tp_fit = tp[sl]
# amplitudes for the peaks
Vp_fit = Vp[sl]
# array indices into full data arrays for the peaks
ipos_fit = imax_fit[sl]
# first peak to be fitted is at 0.
tpk = tp_fit[0]
# get full range data for fitting
# index of peak into the data array
first_peak_ind = ipos_fit[0]
last_peak_ind = ipos_fit[-1]
# slice for data range for fitting into the full array
i_start_data = max(0, first_peak_ind - dl)
i_stop_data = min(last_peak_ind + dl, td.shape[0])
it_fit = slice(i_start_data, i_stop_data)
# find which peaks are in a boundary area
if (it_fit.start == it_fit.stop):
# nothing to fit continue
continue
start_fit_time = td[it_fit][0]
end_fit_time = td[it_fit][-1]
lims.append([start_fit_time, end_fit_time, tpk, it_fit])
# determine of the peaks are in a boundar region
in_boundary[sl] = ((tp_fit - start_fit_time) < boundary) | (
(end_fit_time - tp_fit) < boundary)
# place first peak at 0
tt = td[it_fit] - tpk
Vt = V[it_fit]
# initialize fortran fit
t_peaks = tp_fit - tp_fit[0]
n_peaks = Vp_fit.shape[0]
# initialize vary codes array
vc = np.array(vary_codes_bkg + [1 for v in Vp_fit])
# initalize fit
LF.init_all(alpha(), beta(), t_peaks, n_peaks, poly_order, vc)
# do the fit
chisq = LF.peak_fit(tt, Vt, np.ones_like(Vt), Vt.shape[0])
if (chisq < 0.):
# failed fit
print(' fit ', i, 'failed, chisq = ', chisq)
ifailed += 1
LF.free_all()
continue
# if len(tt)!=0:
# Vline=np.concatenate((Vline, line_shape(tt)))
# tline=np.concatenate((tline, tt+tpk))
# # plot fitting results for check if interval specified
if plot and len(tt) != 0:
pl.plot(tt + tpk, Vt, '.', color='b')
pl.plot(tt + tpk, line_shape(tt), color='m')
# get the amplitudes the first 3 parameters are the bkg.
fitted_A = np.copy(LF.a[bkg_len:])
# get background parameters
bkg = np.copy(LF.a[:bkg_len])
# get covariance matrix
cov = np.copy(LF.covar)
# calculate the error of the amplitudes
sig_fitted_A = np.sqrt(cov.diagonal()[bkg_len:] * chisq)
# get the relevant fit parameters
if (chisq > 0.):
# its_shift.append((tp_fit, sl, Vp_fit, chisq, np.copy(LF.a), np.copy(LF.covar)))
# same the values
A_fit[sl] = fitted_A
sig_A_fit[sl] = sig_fitted_A
bkg_par[sl] = bkg
# free the arrays
LF.free_all()
print(ifailed, ' fits failed out of', i)
# if tmin and tmax:
# return #don't do the second pass if checking the fitting in specified interval
#
#--------------------------------
# get second pass for boundary fit
#--------------------------------
# loop over the peaks
lims_s = []
t_start = time.clock()
N_fitted = 0
# arrays for storing fit results
A_fit_s = np.zeros_like(tp)
sig_A_fit_s = np.zeros_like(tp)
# fit parameters
bkg_par_s = np.zeros(shape=(len(tp), bkg_len))
print('new start time: ', t_start)
# fit shifted set
ifailed1 = 0
for i, ff in enumerate(fg_shift[:]):
N_fitted += fg_shift_n_peaks[i]
if (i % 10 == 0) & (i != 0):
t_current = time.clock()
a_rate = 0.
t_diff = (t_current - t_start)
if (t_diff != 0.):
a_rate = float(N_fitted) / t_diff
print("Fit ", i,
float(N_fitted) / Np * 100., "% completed, time ", t_current,
' rate =', a_rate)
# form a slice for the current peaks
sl = slice(ff[0], ff[1] + 1)
# times for the peaks
tp_fit = tp[sl]
# amplitudes for the peaks
Vp_fit = Vp[sl]
# array indices into full data arrays for the peaks
ipos_fit = imax_fit[sl]
# first peak to be fitted is at 0.
tpk = tp_fit[0]
# get full range data for fitting
# index of peak into the data array
first_peak_ind = ipos_fit[0]
last_peak_ind = ipos_fit[-1]
# slice for data range for fitting into the full array
i_start_data = max(0, first_peak_ind - dl)
i_stop_data = min(last_peak_ind + dl, td.shape[0])
it_fit = slice(i_start_data, i_stop_data)
# find which peaks are in a boundary area
if (it_fit.start == it_fit.stop):
# nothing to fit continue
continue
start_fit_time = td[it_fit][0]
end_fit_time = td[it_fit][-1]
lims_s.append([start_fit_time, end_fit_time, tpk, it_fit])
# place first peak at 0
tt = td[it_fit] - tpk
Vt = V[it_fit]
# initialize fortran fit
t_peaks = tp_fit - tp_fit[0]
n_peaks = Vp_fit.shape[0]
# initialize vary codes array
vc = np.array(vary_codes_bkg + [1 for v in Vp_fit])
# initalize fit
LF.init_all(alpha(), beta(), t_peaks, n_peaks, poly_order, vc)
# do the fit
chisq = LF.peak_fit(tt, Vt, np.ones_like(Vt), Vt.shape[0])
if (chisq < 0.):
# failed fit
print(' fit ', i, 'failed, chisq = ', chisq)
ifailed1 += 1
LF.free_all()
continue
# plot result if interval specified
if plot_s:
pl.figure()
if len(tt) != 0:
pl.plot(tt + tpk, Vt, '.', color='b')
pl.plot(tt + tpk, line_shape(tt), color='m')
pl.title('Shifted fitting groups')
# save the parameters
# get the amplitudes the first 3 parameters are the bkg.
fitted_A = np.copy(LF.a[bkg_len:])
# get background parameters
bkg = np.copy(LF.a[:bkg_len])
# get covariance matrix
cov = np.copy(LF.covar)
# calculate the error of the amplitudes
sig_fitted_A = np.sqrt(cov.diagonal()[bkg_len:] * chisq)
# get the relevant fit parameters
if (chisq > 0.):
# its_shift.append((tp_fit, sl, Vp_fit, chisq, np.copy(LF.a), np.copy(LF.covar)))
# same the values
A_fit_s[sl] = fitted_A
sig_A_fit_s[sl] = sig_fitted_A
bkg_par_s[sl] = bkg
# free the arrays
LF.free_all()
print(ifailed1, ' fits failed out of', i)
#--------------------------------
# copy results of those peaks that are in the boundaryregion from the 2nd fit
# in_boundary is try for all peaks that lie in a boundary region
# need to get the fit results from the shifted fit for those peaks e.g. for the indices
#--------------------------------
A_fit[in_boundary] = A_fit_s[in_boundary]
sig_A_fit[in_boundary] = sig_A_fit_s[in_boundary]
bkg_par[in_boundary] = bkg_par_s[in_boundary]
#---------------------------------------------
# save the data as numpy compressed data files
#---------------------------------------------
if not save:
print('no results saved!')
else:
# save the fitted data
o_file = self.var['res_dir'] + "fit_results_" + str(
self.par['shot']) + "_{0:5.3f}_{1:5.3f}_{2:d}.npz".format(
tmin, tmax, self.par['channel'])
if not os.path.exists(os.path.dirname(o_file)):
os.makedirs(os.path.dirname(o_file))
if os.path.isfile(o_file):
o_file = self.var['res_dir'] + "fit_results_" + str(
self.par['shot']) + "_{0:5.3f}_{1:5.3f}_{2:d}".format(
tmin, tmax, self.par['channel']) + time.strftime(
'%d_%m_%Y_%H_%M_%S') + ".npz"
n_lines = tp.shape[0]
np.savez_compressed(o_file,
t=tp,
V=Vp,
A=A_fit,
sig_A=sig_A_fit,
bkg=bkg_par)
print("Wrote : ", n_lines, " lines to the output file")
#save fitted lines to the file
# np.savez_compressed(self.var['res_dir'] + "fit_line_"+ str(self.par['shot']) + ".npz", Vline=Vline, tline=tline)
# print "saved line"
X = list()
Y = list()
for i in listdir(link):
path = link + i + '/'
faces = load_face(path)
label = [i for _ in range(len(faces))]
X.extend(faces)
Y.extend(label)
return X, Y
link_train = '/home/face_id/Facenet/VN_train/'
link_test = '/home/face_id/Facenet/VN_test/'
train_x, train_y = load_data(link_train)
test_x, test_y = load_data(link_test)
savez_compressed('/home/face_id/Facenet/data.npz', train_x, train_y, test_x,
test_y)
data = numpy.load('/home/face_id/Facenet/data.npz')
train_x, train_y, test_x, test_y = data['arr_0'], data['arr_1'], data[
'arr_2'], data['arr_3']
model = load_model('/home/face_id/Facenet/facenet_keras.h5')
def get_embedding(model, face_pixels):
face_pixels = face_pixels.astype('float32')
mean = face_pixels.mean()
std = face_pixels.std()
face_pixels = (face_pixels - mean) / std
samples = numpy.expand_dims(face_pixels, axis=0)
a = model.predict(samples)
return a[0]
# Normalize input so we can train ANN with it. # Will be converted back to integers for SNN layer. x_train = x_train / 255 x_test = x_test / 255 # Add a channel dimension. axis = 1 if keras.backend.image_data_format() == 'channels_first' else -1 x_train = np.expand_dims(x_train, axis) x_test = np.expand_dims(x_test, axis) # One-hot encode target vectors. y_train = to_categorical(y_train, 10) y_test = to_categorical(y_test, 10) # Save dataset so SNN toolbox can find it. np.savez_compressed(os.path.join(path_wd, 'x_test'), x_test) np.savez_compressed(os.path.join(path_wd, 'y_test'), y_test) # SNN toolbox will not do any training, but we save a subset of the training # set so the toolbox can use it when normalizing the network parameters. np.savez_compressed(os.path.join(path_wd, 'x_norm'), x_train[::10]) # CREATE ANN # ############## # This section creates a simple CNN using Keras, and trains it # with backpropagation. There are no spikes involved at this point. input_shape = x_train.shape[1:] input_layer = Input(input_shape) layer = Conv2D(filters=16, kernel_size=(5, 5), strides=(2, 2))(input_layer)
def save_weights(self, path):
np.savez_compressed('e' + str(self.nb_epoch) + '_w1.npz', self.W[1])
np.savez_compressed('e' + str(self.nb_epoch) + '_w2.npz', self.W[2])
np.savez_compressed('e' + str(self.nb_epoch) + '_w3.npz', self.W[3])
np.savez_compressed('e' + str(self.nb_epoch) + '_b1.npz', self.b[1])
np.savez_compressed('e' + str(self.nb_epoch) + '_b2.npz', self.b[2])
np.savez_compressed('e' + str(self.nb_epoch) + '_b3.npz', self.b[3])
import gridfs
db = pymongo.MongoClient().maxilafacial
fileDB = gridfs.GridFS(db)
sample_size = 4096
if __name__ == "__main__":
#Generate Training Data Here!
patients = db.patient.find({})
for element in patients:
patientID = element["_id"]
if not element['status']: continue
train_data = utils.make_trainin_data(patientID,
size=100,
sample_size=sample_size)
input_data = []
gt_data = []
for data in train_data:
input_data.append(data['input'])
gt_data.append(data['gt'])
#Save By Patient ID
np.savez_compressed(os.path.join('processed', str(patientID)),
input=input_data,
gt=gt_data)
data_dict = json.load(f)
elif data_type == 'val':
with open(os.path.join(val_dir, 'AgriculturalDisease_validation_annotations.json'), 'r') as f:
data_dict = json.load(f)
else:
raise Exception('[KF ERROR] data_type not correct: train or val')
print('')
print('============================================================')
print(' LOAD DATA')
print('============================================================')
print('[KF INFO] Total %s sample to be loaded: ' % data_type, len(data_dict))
print("[KF INFO] Loading %s data ..." % data_type)
for item in data_dict:
image = load_img(os.path.join(data_dir, 'images', item['image_id']), target_size=image_shape)
image_np = img_to_array(image)
data.append(image_np)
labels.append(item['disease_class'])
print(os.path.join(data_dir, 'images', item['image_id']), " is loaded.")
# Save data and labels into npz file
if not os.path.exists(output_dir):
os.makedirs(output_dir)
save_name = data_type + '-' + img_size
print("[KF INFO] Saving %s data and labels into %s.npz ..." % (data_type, save_name))
np.savez_compressed(os.path.join(output_dir, save_name), data=data, labels=labels)
print("[KF INFO] Data and labels are saved successfully!")
def main():
args = check_argv()
if not path.isdir(args.data_dir):
os.makedirs(args.data_dir)
print(datetime.datetime.now())
# Training data: ICASSP 2015
scp_fn = path.join(data_base_dir, "data/clusters_gt/clusters_gt.mfcc.scp")
print("Reading:", scp_fn)
ark_dict = read_kaldi_ark_from_scp(scp_fn, data_base_dir)
ids = sorted(ark_dict.keys())
mats = [ark_dict[i].T for i in ids]
train_mean = np.mean(np.hstack(mats).flatten())
mats = pad_images_width(mats, n_padded, train_mean) - train_mean
print("Padded and normalized", mats.shape[0], "data instances")
npz_fn = path.join(args.data_dir, "swbd.train.npz")
print("Writing:", npz_fn)
npz_dict = {}
for i, utt_id in enumerate(ids):
npz_dict[utt_id] = mats[i]
np.savez_compressed(npz_fn, **npz_dict)
train_mean_fn = path.join(args.data_dir, "train.mean")
print("Writing:", train_mean_fn)
with open(train_mean_fn, "w") as f:
f.write(str(train_mean) + "\n")
# plot_fn = path.join(args.data_dir, "train_example.png")
# print("Saving:", plot_fn)
# image = Image.fromarray(plotting.array_to_pixels(mats[0]))
# image.save(plot_fn)
# # Training data: ASRU 2013
# scp_fn = path.join(data_base_dir, "data/clusters_gt_asru13/clusters_gt_asru13.mfcc.scp")
# print "Reading:", scp_fn
# ark_dict = read_kaldi_ark_from_scp(scp_fn, data_base_dir)
# ids = sorted(ark_dict.keys())
# mats = [ark_dict[i].T for i in ids]
# train_mean_asru13 = np.mean(np.hstack(mats).flatten())
# mats = pad_images_width(mats, n_padded, train_mean_asru13) - train_mean_asru13
# print "Padded and normalized", mats.shape[0], "data instances"
# npz_fn = path.join(args.data_dir, "swbd.train_asru13.npz")
# print "Writing:", npz_fn
# npz_dict = {}
# for i, utt_id in enumerate(ids):
# npz_dict[utt_id] = mats[i]
# np.savez_compressed(npz_fn, **npz_dict)
# train_mean_fn = path.join(args.data_dir, "train_asru13.mean")
# print "Writing:", train_mean_fn
# with open(train_mean_fn, "w") as f:
# f.write(str(train_mean_asru13) + "\n")
# plot_fn = path.join(args.data_dir, "train_asru13_example.png")
# print "Saving:", plot_fn
# image = Image.fromarray(plotting.array_to_pixels(mats[0]))
# image.save(plot_fn)
# Test data
scp_fn = path.join(data_base_dir,
"data/samediff_test/samediff_test.mfcc.scp")
print("Reading:", scp_fn)
ark_dict = read_kaldi_ark_from_scp(scp_fn, data_base_dir)
ids = sorted(ark_dict.keys())
mats = [ark_dict[i].T for i in ids]
mats = pad_images_width(mats, n_padded, train_mean) - train_mean
print("Padded and normalized", mats.shape[0], "data instances")
npz_fn = path.join(args.data_dir, "swbd.test.npz")
print("Writing:", npz_fn)
npz_dict = {}
for i, utt_id in enumerate(ids):
npz_dict[utt_id] = mats[i]
np.savez_compressed(npz_fn, **npz_dict)
# # Development data
scp_fn = path.join(data_base_dir,
"data/samediff_dev/samediff_dev.mfcc.scp")
print("Reading:", scp_fn)
ark_dict = read_kaldi_ark_from_scp(scp_fn, data_base_dir)
ids = sorted(ark_dict.keys())
mats = [ark_dict[i].T for i in ids]
mats = pad_images_width(mats, n_padded, train_mean) - train_mean
print("Padded and normalized", mats.shape[0], "data instances")
npz_fn = path.join(args.data_dir, "swbd.dev.npz")
print("Writing:", npz_fn)
npz_dict = {}
for i, utt_id in enumerate(ids):
npz_dict[utt_id] = mats[i]
np.savez_compressed(npz_fn, **npz_dict)
print(datetime.datetime.now())
def hmm_inference(self, models, test_X, test_y):
EMOTIONS = self.emotions
RR_1 = {
emotion: {emotion: 0
for emotion in EMOTIONS}
for emotion in EMOTIONS
}
RR_2 = {
emotion: {emotion: 0
for emotion in EMOTIONS}
for emotion in EMOTIONS
}
RR_cum = {
emotion: {emotion: 0
for emotion in EMOTIONS}
for emotion in EMOTIONS
}
gt_counts = {emotion: 0 for emotion in EMOTIONS}
pred_pos_1 = {
emotion: {emotion: 0
for emotion in EMOTIONS}
for emotion in EMOTIONS
} # Number predicted at position 1
pred_pos_2 = {
emotion: {emotion: 0
for emotion in EMOTIONS}
for emotion in EMOTIONS
} # Number predicted at position 2
print('Predicting emotion from model...')
for i, vid in enumerate(test_X):
gt = test_y[i]
print("Predicting emotion for gt " + gt)
gt_counts[gt] += 1
predictions = {
emotion: models[emotion].viterbi(vid)
for emotion in EMOTIONS
}
predictions = [predictions[emotion][0] for emotion in EMOTIONS]
emotion_pred_0 = EMOTIONS[np.argmax(predictions)]
print("Predicted emotion is: " + emotion_pred_0)
predictions[np.argmax(predictions)] = 0
emotion_pred_1 = EMOTIONS[np.argmax(predictions)]
print("Second predicted emotion is: " + emotion_pred_1)
RR_1[gt][emotion_pred_0] += 1
RR_2[gt][emotion_pred_1] += 1
pred_pos_1[gt][emotion_pred_0] += 1
pred_pos_2[gt][emotion_pred_1] += 1
for emotion1 in RR_1:
for emotion2 in RR_1[emotion1]:
RR_1[emotion1][
emotion2] = RR_1[emotion1][emotion2] / gt_counts[emotion1]
RR_2[emotion1][
emotion2] = RR_2[emotion1][emotion2] / gt_counts[emotion1]
RR_cum[emotion1][emotion2] = RR_1[emotion1][emotion2] + RR_2[
emotion1][emotion2]
print("Writing recognition results to " + self.model_results_path +
"...")
np.savez_compressed(self.model_results_path,
RR_1=RR_1,
RR_2=RR_2,
RR_cum=RR_cum,
pred_pos_1=pred_pos_1,
pred_pos_2=pred_pos_2)
print('Done')
def test(self):
epoch = self.scheduler1.last_epoch + 1
self.ckp.write_log('\nEvaluation:')
self.ckp.add_log(torch.zeros(1, len(self.scale)))
self.model1.eval()
if (self.args.nmodels == 2):
self.ckp2.write_log('\nEvaluation:')
self.ckp2.add_log(torch.zeros(1, len(self.scale)))
self.model2.eval()
timer_test = utility.timer()
with torch.no_grad():
for idx_scale, scale in enumerate(self.scale):
eval_acc = 0
if (self.args.nmodels == 2):
eval_acc2 = 0
self.loader_test.dataset.set_scale(idx_scale)
tqdm_test = tqdm(self.loader_test, ncols=80)
n_test_data = len(self.loader_test.dataset)
print("n_test_data", n_test_data)
Test_pred_mat_HR = np.zeros((n_test_data, 3, 111, 111))
Test_pred_mat_LR = np.zeros((n_test_data, 3, 111, 111))
Test_pred_mat_SR = np.zeros((n_test_data, 3, 111, 111))
Test_pred_mat_Limg = np.zeros((n_test_data, 3, 111, 111))
if (self.args.nmodels == 2):
Test_pred_mat_HR_2 = np.zeros((n_test_data, 3, 111, 111))
Test_pred_mat_LR_2 = np.zeros((n_test_data, 3, 111, 111))
Test_pred_mat_SR_2 = np.zeros((n_test_data, 3, 111, 111))
#Test_pred_mat_Limg_2 = np.zeros((n_test_data,3,111,111))
k = 0
for idx_img, (lr, hr, lr2, hr2, filename,
_) in enumerate(tqdm_test):
filename = filename[0]
#no_eval = (hr.item() == -1)
no_eval = False
# print ("before prep", lr.dtype,hr.dtype)
# print ("before prep", lr.max(),lr.min(),hr.max(),hr.min())
if not no_eval:
if (self.args.model == 'MWCNN'):
lr, hr = self.prepare_test([lr, hr])
else:
lr, hr = self.prepare([lr, hr])
else:
if (self.args.model == 'MWCNN'):
lr = self.prepare_test([lr])[0]
else:
lr = self.prepare([lr])[0]
# print ("After prep", lr.dtype,hr.dtype)
# print ("After prep", lr.max(),lr.min(),hr.max(),hr.min())
sr1 = self.model1(lr, idx_scale)
if (self.args.model == 'MWCNN'):
hr = hr[:, :, 1:, 1:]
lr = lr[:, :, 1:, 1:]
if (self.args.nmodels == 2):
if (self.args.Test_feed_model1_out):
###### the SR from Model 1 id used as LR for model 2
if not no_eval:
#lr2, hr2 = self.prepare([sr1, hr2])
#lr2, hr2 = self.prepare([lr2, hr2])
lr2 = sr1.to(torch.device('cuda'))
lr2 = utility.quantize(lr2,
self.args.rgb_range)
hr2 = hr2.to(torch.device('cuda'))
else:
if (self.args.model == 'MWCNN'):
lr2 = self.prepare_test([sr1])[0]
else:
lr2 = self.prepare([lr2])[0]
else:
if not no_eval:
if (self.args.model == 'MWCNN'):
lr2, hr2 = self.prepare_test([lr2, hr2])
else:
lr2, hr2 = self.prepare([lr2, hr2])
else:
if (self.args.model == 'MWCNN'):
lr2 = self.prepare_test([lr2])[0]
else:
lr2 = self.prepare([lr2])[0]
#lr2 = sr1
#sr1_prep = self.prepare([sr1])[0]
sr2 = self.model2(lr2, idx_scale)
if (self.args.model == 'MWCNN'):
#hr2 = hr[:,:,1:,1:]
lr2 = lr2[:, :, 1:, 1:]
sr2 = sr2[:, :, 1:, 1:]
# print ("After eval", sr.dtype)
# print ("After eval", sr.max(),sr.min())
if (self.args.model == 'MWCNN'):
sr1 = sr1[:, :, 1:, 1:]
sr1 = utility.quantize(sr1, self.args.rgb_range)
Test_pred_mat_HR[k * 250:(k + 1) * 250, :, :, :] = hr
Test_pred_mat_LR[k * 250:(k + 1) * 250, :, :, :] = lr
Test_pred_mat_SR[k * 250:(k + 1) * 250, :, :, :] = sr1
Test_pred_mat_Limg[k * 250:(k + 1) * 250:, :, :] = hr2
save_list = [sr1]
if (self.args.nmodels == 2):
sr2 = utility.quantize(sr2, self.args.rgb_range)
# print ("After quantize", sr.dtype)
# print ("After quantize", sr.max(),sr.min())
Test_pred_mat_HR_2[k * 250:(k + 1) *
250, :, :, :] = hr2
Test_pred_mat_LR_2[k * 250:(k + 1) *
250, :, :, :] = lr2
Test_pred_mat_SR_2[k * 250:(k + 1) *
250, :, :, :] = sr2
#Test_pred_mat_Limg_2[k,:,:,:] = hr2
#print ("TEST-Data shape-LR",lr.size())
#print ("TEST-Data shape-HR",hr.size())
#print ("TEST-Data shape-SR",sr.size())
save_list2 = [sr2]
if not no_eval:
eval_acc += utility.calc_psnr(
sr1,
hr,
scale,
self.args.rgb_range,
benchmark=self.loader_test.dataset.benchmark)
save_list.extend([lr, hr])
if (self.args.nmodels == 2):
eval_acc2 += utility.calc_psnr(
sr2,
hr2,
scale,
self.args.rgb_range,
benchmark=self.loader_test.dataset.benchmark)
#print ("eval_acc",eval_acc,"eval_acc2",eval_acc2)
save_list2.extend([lr2, hr2])
if self.args.save_results:
self.ckp.save_results(filename, save_list, scale)
if (self.args.nmodels == 2):
self.ckp2.save_results(filename, save_list2, scale)
k = k + 1
dir_save = '/gpfs/jlse-fs0/users/sand33p/stronglensing/Image_Enhancement/EDSR_MWCNN/experiment/' + self.args.save
filename_save_HR = '{}/results/Array_HR.npz'.format(dir_save)
filename_save_LR = '{}/results/Array_LR.npz'.format(dir_save)
filename_save_SR = '{}/results/Array_SR.npz'.format(dir_save)
filename_save_Limg = '{}/results/Array_Limg.npz'.format(
dir_save)
np.savez_compressed(filename_save_HR, X=Test_pred_mat_HR)
np.savez_compressed(filename_save_LR, X=Test_pred_mat_LR)
np.savez_compressed(filename_save_SR, X=Test_pred_mat_SR)
np.savez_compressed(filename_save_Limg, X=Test_pred_mat_Limg)
if not self.args.test_only:
print("self.it_ckp", self.it_ckp)
with h5py.File(self.HDF5_file_model1_loc, 'a') as hf:
hf["Array_HR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_HR
hf["Array_LR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_LR
hf["Array_SR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_SR
hf["Array_Limg"][
self.it_ckp, :, :, :, :] = Test_pred_mat_Limg
hf.close()
self.ckp.log[-1, idx_scale] = eval_acc / len(self.loader_test)
best = self.ckp.log.max(0)
self.ckp.write_log(
'[{} x{}]\tPSNR-ckp: {:.3f} (Best: {:.3f} @epoch {})'.
format(self.args.data_test, scale, self.ckp.log[-1,
idx_scale],
best[0][idx_scale], best[1][idx_scale] + 1))
if (self.args.nmodels == 2):
dir_save = '/gpfs/jlse-fs0/users/sand33p/stronglensing/Image_Enhancement/EDSR_MWCNN/experiment/' + self.args.save
filename_save_HR_2 = '{}/results/Array_HR_2.npz'.format(
dir_save)
filename_save_LR_2 = '{}/results/Array_LR_2.npz'.format(
dir_save)
filename_save_SR_2 = '{}/results/Array_SR_2.npz'.format(
dir_save)
#filename_save_Limg_2 = '{}/results/Array_Limg_2.npz'.format(dir_save)
np.savez_compressed(filename_save_HR_2,
X=Test_pred_mat_HR_2)
np.savez_compressed(filename_save_LR_2,
X=Test_pred_mat_LR_2)
np.savez_compressed(filename_save_SR_2,
X=Test_pred_mat_SR_2)
if not self.args.test_only:
with h5py.File(self.HDF5_file_model2_loc, 'a') as hf2:
hf2["Array_HR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_HR_2
hf2["Array_LR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_LR_2
hf2["Array_SR"][
self.it_ckp, :, :, :, :] = Test_pred_mat_SR_2
hf2.close()
self.ckp2.log[-1, idx_scale] = eval_acc2 / len(
self.loader_test)
best2 = self.ckp2.log.max(0)
self.ckp2.write_log(
'[{} x{}]\tPSNR-ckp2: {:.3f} (Best: {:.3f} @epoch {})'.
format(self.args.data_test, scale,
self.ckp2.log[-1, idx_scale],
best2[0][idx_scale], best2[1][idx_scale] + 1))
self.ckp.write_log('Total time: {:.2f}s\n'.format(timer_test.toc()),
refresh=True)
if (self.args.nmodels == 2):
self.ckp2.write_log('Total time: {:.2f}s\n'.format(
timer_test.toc()),
refresh=True)
if not self.args.test_only:
#save(self, trainer, epoch, is_best=False,model=trainer.model1,loss=trainer.loss1,optimizer=trainer.optimizer1,model_name='model1'):
self.ckp.save(self,
epoch,
is_best=(best[1][0] + 1 == epoch),
model=self.model1,
loss=self.loss1,
optimizer=self.optimizer1,
model_name='model1')
if ((self.args.nmodels == 2) & (self.args.numloss == 2)):
if (self.use_two_opt):
self.ckp2.save(self,
epoch,
is_best=(best2[1][0] + 1 == epoch),
model=self.model2,
loss=self.loss2,
optimizer=self.optimizer2,
model_name='model2')
else:
self.ckp2.save(self,
epoch,
is_best=(best2[1][0] + 1 == epoch),
model=self.model2,
loss=self.loss2,
optimizer=self.optimizer1,
model_name='model2')
#elif (self.args.nmodels == 2):
# self.ckp2.save(self, epoch, is_best=(best2[1][0] + 1 == epoch),model=self.model2,loss=self.loss1,optimizer=self.optimizer1,model_name='model2')
if not self.args.test_only:
self.it_ckp = self.it_ckp + 1
(poisson_mag + 6) * (np.arange(ramp_length) /
poisson_mag) ** 0.5).astype(int)
np.clip(v1, 0, None, v1)
#plt.plot(v1)
#plt.show(block=True)
volume_noise = np.random.normal(scale=0.05, size=(p, w, w))
indices_pln = (horizon[np.newaxis, ...] +
np.arange(m, p - M)[:, np.newaxis, np.newaxis])
indices_row = np.arange(w)[np.newaxis, :, np.newaxis]
indices_col = np.arange(w)[np.newaxis, np.newaxis, :]
index_vol = (indices_pln, indices_row, indices_col)
volume_noise[index_vol] += v0[:, np.newaxis, np.newaxis]
#plt.imshow(volume_noise[:, 50, :])
#plt.show(block=True)
#plt.hist(volume_noise.ravel(), bins=256)
#plt.show(block=True)
volume_noise2 = np.random.poisson(2, size=(p, w, w)) - 2
volume_noise2[index_vol] += v1[:, np.newaxis, np.newaxis]
np.clip(volume_noise2, 0, None, volume_noise2)
#plt.imshow(volume_noise2[:, 50, :])
#plt.show(block=True)
MM = max(abs(M), abs(m))
volume0 = volume_noise[MM:-MM, :, :]
_, bins = np.histogram(volume0, bins=255)
volume0 = np.clip(np.digitize(volume0, bins), 0, 255).astype(np.uint8)
volume1 = volume_noise2[MM:-MM, :, :].astype(np.uint8)
np.savez_compressed('geo.npz', strata=volume0, density=volume1)
test_acc = th.eq(test_pred[test_mask],
labels[test_mask]).float().mean().item()
test_hidden_features = net.gat1(g, features).cpu().numpy()
final_train_pred = test_pred[train_mask].cpu().numpy()
final_val_pred = test_pred[val_mask].cpu().numpy()
final_test_pred = test_pred[test_mask].cpu().numpy()
results_dict = vars(args)
results_dict['test_loss'] = test_loss
results_dict['test_acc'] = test_acc
results_dict['actual_epochs'] = 1 + epoch
results_dict['val_acc_max'] = vacc_mx
results_dict['val_loss_min'] = vlss_mn
results_dict['total_time'] = sum(dur)
with open(os.path.join('runs', f'{args.model}_{args.run_id}_results.txt'),
'w') as outfile:
outfile.write(json.dumps(results_dict) + '\n')
np.savez_compressed(os.path.join(
'runs', f'{args.model}_{args.run_id}_hidden_features.npz'),
hidden_features=test_hidden_features)
np.savez_compressed(os.path.join(
'runs', f'{args.model}_{args.run_id}_final_train_predictions.npz'),
final_train_predictions=final_train_pred)
np.savez_compressed(os.path.join(
'runs', f'{args.model}_{args.run_id}_final_val_predictions.npz'),
final_val_predictions=final_val_pred)
np.savez_compressed(os.path.join(
'runs', f'{args.model}_{args.run_id}_final_test_predictions.npz'),
final_test_predictions=final_test_pred)
def learn(self,
total_timesteps,
callback=None,
log_interval=1,
tb_log_name="PPO2",
reset_num_timesteps=True):
# Transform to callable if needed
self.learning_rate = get_schedule_fn(self.learning_rate)
self.cliprange = get_schedule_fn(self.cliprange)
cliprange_vf = get_schedule_fn(self.cliprange_vf)
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
if self.savpath is not None:
rewardlist = []
obslist = []
actionlist = []
with SetVerbosity(self.verbose), TensorboardWriter(self.graph, self.tensorboard_log, tb_log_name, new_tb_log) \
as writer:
self._setup_learn()
runner = Runner(env=self.env,
model=self,
n_steps=self.n_steps,
gamma=self.gamma,
lam=self.lam)
t_first_start = time.time()
n_updates = total_timesteps // self.n_batch
for update in range(1, n_updates + 1):
assert self.n_batch % self.nminibatches == 0, (
"The number of minibatches (`nminibatches`) "
"is not a factor of the total number of samples "
"collected per rollout (`n_batch`), "
"some samples won't be used.")
batch_size = self.n_batch // self.nminibatches
t_start = time.time()
frac = 1.0 - (update - 1.0) / n_updates
lr_now = self.learning_rate(frac)
cliprange_now = self.cliprange(frac)
cliprange_vf_now = cliprange_vf(frac)
# true_reward is the reward without discount
obs, returns, masks, actions, values, neglogpacs, states, ep_infos, true_reward = runner.run(
)
if self.savpath is not None:
rewardlist += [true_reward]
obslist += [obs]
actionlist += [actions]
self.num_timesteps += self.n_batch
self.ep_info_buf.extend(ep_infos)
mb_loss_vals = []
if states is None: # nonrecurrent version
update_fac = self.n_batch // self.nminibatches // self.noptepochs + 1
inds = np.arange(self.n_batch)
for epoch_num in range(self.noptepochs):
np.random.shuffle(inds)
for start in range(0, self.n_batch, batch_size):
timestep = self.num_timesteps // update_fac + (
(self.noptepochs * self.n_batch + epoch_num *
self.n_batch + start) // batch_size)
end = start + batch_size
mbinds = inds[start:end]
slices = (arr[mbinds]
for arr in (obs, returns, masks, actions,
values, neglogpacs))
mb_loss_vals.append(
self._train_step(
lr_now,
cliprange_now,
*slices,
writer=writer,
update=timestep,
cliprange_vf=cliprange_vf_now))
else: # recurrent version
update_fac = self.n_batch // self.nminibatches // self.noptepochs // self.n_steps + 1
assert self.n_envs % self.nminibatches == 0
env_indices = np.arange(self.n_envs)
flat_indices = np.arange(self.n_envs *
self.n_steps).reshape(
self.n_envs, self.n_steps)
envs_per_batch = batch_size // self.n_steps
for epoch_num in range(self.noptepochs):
np.random.shuffle(env_indices)
for start in range(0, self.n_envs, envs_per_batch):
timestep = self.num_timesteps // update_fac + (
(self.noptepochs * self.n_envs + epoch_num *
self.n_envs + start) // envs_per_batch)
end = start + envs_per_batch
mb_env_inds = env_indices[start:end]
mb_flat_inds = flat_indices[mb_env_inds].ravel()
slices = (arr[mb_flat_inds]
for arr in (obs, returns, masks, actions,
values, neglogpacs))
mb_states = states[mb_env_inds]
mb_loss_vals.append(
self._train_step(
lr_now,
cliprange_now,
*slices,
update=timestep,
writer=writer,
states=mb_states,
cliprange_vf=cliprange_vf_now))
loss_vals = np.mean(mb_loss_vals, axis=0)
t_now = time.time()
fps = int(self.n_batch / (t_now - t_start))
if writer is not None:
total_episode_reward_logger(
self.episode_reward,
true_reward.reshape((self.n_envs, self.n_steps)),
masks.reshape((self.n_envs, self.n_steps)), writer,
self.num_timesteps)
if self.verbose >= 1 and (update % log_interval == 0
or update == 1):
explained_var = explained_variance(values, returns)
logger.logkv("serial_timesteps", update * self.n_steps)
logger.logkv("n_updates", update)
logger.logkv("total_timesteps", self.num_timesteps)
logger.logkv("fps", fps)
logger.logkv("explained_variance", float(explained_var))
if len(self.ep_info_buf) > 0 and len(
self.ep_info_buf[0]) > 0:
logger.logkv(
'ep_reward_mean',
safe_mean([
ep_info['r'] for ep_info in self.ep_info_buf
]))
logger.logkv(
'ep_len_mean',
safe_mean([
ep_info['l'] for ep_info in self.ep_info_buf
]))
logger.logkv('time_elapsed', t_start - t_first_start)
for (loss_val, loss_name) in zip(loss_vals,
self.loss_names):
logger.logkv(loss_name, loss_val)
logger.dumpkvs()
if callback is not None:
# Only stop training if return value is False, not when it is None. This is for backwards
# compatibility with callbacks that have no return statement.
if callback(locals(), globals()) is False:
break
if self.savpath is not None:
np.savez_compressed(self.savpath,
rewards=rewardlist,
observations=obslist,
actions=actionlist)
return self
# Generates face landmarks one-by-one
# This part can be modified to predict the whole sequence at one, but may introduce discontinuities
for i in tqdm(range(upper_limit)):
cur_features = np.zeros((1, num_frames, features.shape[2]))
if i + 1 > 75:
lower = i + 1 - 75
cur_features[:, -i - 1:, :] = features[:, lower:i + 1, :]
pred = model.predict(cur_features)
generated = np.append(generated,
np.reshape(pred[0, -1, :], (1, num_features_Y)),
axis=0)
# Shift the array to remove the delay
generated = generated[trainDelay:, :]
tmp = generated[-1:, :]
for _ in range(trainDelay):
generated = np.append(generated, tmp, axis=0)
if len(generated.shape) < 3:
generated = np.reshape(
generated, (generated.shape[0], int(generated.shape[1] / 2), 2))
keypoints = generated * 100
print(keypoints.shape)
data = asarray(keypoints)
savez_compressed(output_path + '/data.npz', data)
no_workers,
in_dim,
data_func,
data_type=data_type,
seed=seed) for i in range(no_workers)
]
i = 0
current_params = ray.get(ps.pull.remote(all_keys))
path_prefix = 'logs/distributed_training/'
path = path_prefix + 'gra_pvi_sync_%s_data_%s_seed_%d_no_workers_%d_damping_%.3f/' % (
dataset, data_type, seed, no_workers, damping)
if not os.path.exists(path):
os.makedirs(path)
np.savez_compressed(path + 'params_interval_%d.npz' % i,
n1=current_params[0],
n2=current_params[1])
time_fname = path + 'train_time.txt'
time_file = open(time_fname, 'w', 0)
time_file.write('%.4f\n' % 0)
tracker_file = "logs/gra_indiv_terms.txt"
if os.path.exists(tracker_file):
os.remove(tracker_file)
while i < no_intervals:
start_time = time.time()
###########
deltas = [
worker.get_delta.remote(current_params, damping=damping)
for worker in workers
np.unique(names)
ints = np.array([3, 3, 3, 2, 2, 1, 1, 4, 4])
np.unique(ints)
values = np.array([6, 0, 0, 3, 2, 5, 6])
np.in1d(values, [2, 3, 6])
arr = np.arange(10)
np.save('some_array', arr)
np.load('some_array.npy')
np.savez('array_archive.npz', a = arr, b = arr)
arch = np.load('array_archive.npz')
arch['a']
np.savez_compressed('arrays_compressed.npz', a = arr, b = arr)
####################################### Linear Algebra #############################
x = np.array([[1., 2., 3.], [4., 5., 6.]])
y = np.array([[6., 23.], [-1, 7], [8, 9]])
x
y
x.dot(y)
np.dot(x, np.ones(3))
x @ np.ones(3)
from numpy.linalg import inv, qr
def bam2clusters(bam, fasta, outdir, minSize=2000, mapq=10, threads=4, dpi=100, upto=0, nchr=0,
verbose=1, minchr=3, method="ward", contigs=[]):
"""Return clusters computed from from windowSizes"""
logger("=== Clustering ===")
outbase = os.path.join(outdir, "chr_%s"%nchr if nchr else "auto")
# link with auto
if nchr and os.path.isfile(os.path.join(outdir, "auto.npz")) and not os.path.isfile(outbase+".npz"):
logger(" reusing %s/auto.* ..."%outbase)
for ext in (".npz", ".windows.tab.gz", ".distance.params"):
os.system("ln -s auto%s %s%s"%(ext, outbase, ext))
# load clusters
fname = outbase + ".clusters.tab"
if os.path.isfile(fname):
clusters = [l[:-1].split('\t') for l in open(fname)]
logger(" %s clusters loaded from %s."%(len(clusters), fname))
return clusters
# get windows
minSize, windows, chr2window, contig2size = contigs2windows(fasta, minSize, verbose, contigs=contigs)
# generate missing handles
bin_chr, bin_position = [], []
for c, s, e in windows:
bin_chr.append(c)
bin_position.append((s, e))
bin_chr = np.array(bin_chr)
bin_position = np.array(bin_position)
if not os.path.isfile(outbase+".npz"): #.balanced
# get array from bam
logger("Parsing BAM...")
d, c2dists = bam2array(windows, contig2size, chr2window, bam, mapq, upto, threads=threads, minSize=minSize)
# save windows, array and plot
logger("Saving array...")
with gzip.open(outbase + ".windows.tab.gz", "w") as out:
out.write("\n".join("\t".join(map(str, w)) for w in windows)+"\n")
with open(outbase+".npz", "w") as out:
np.savez_compressed(out, d)
# estimate parameters
params = estimate_distance_parameters(outbase, contig2size=contig2size, c2dists=c2dists, windowSize=minSize,
contigs=contigs)
# load from file
else:
npy = np.load(outbase+".npz")
d = npy[npy.files[0]]
params = pickle.load(open(outbase+".distance.params"))
# make symmetric
d += d.T - np.diag(d.diagonal())
#d = normalize(d)
#params = estimate_distance_parameters(outbase, contig2size=contig2size, c2dists=c2dists, windowSize=minSize)
# get clusters on transformed matrix
transform = lambda x: distance_func(x+1, *params); print "dist"
#transform = lambda x: np.sum(x+1.) / (1e6*(x+1)); print "sum / xM"
#transform = lambda x: np.max(x+1., axis=0) / (x+1) - 1; print "max0/x - 1"
#transform = lambda x: np.max(x+1.) / (x+1) - 1; print "max/x - 1"
#transform = lambda x: np.log(np.max(x+1))-np.log(x+1); print "log max x - log x"
#transform = lambda x: np.log(np.max(x+1, axis=0))-np.log(x+1); print "log max x0 - log x"
clusters = cluster_contigs(outbase, transform(d), bin_chr, bin_position, dpi=dpi, minchr=minchr, nchr=nchr)
# skip empty clusters
clusters = filter(lambda x: x, clusters)
totsize = contigs = 0
outfn = outbase+".clusters.tab"
logger("Reporting %s clusters to %s ..."%(len(clusters), outfn))
with open(outfn, "w") as out:
for i, cluster in enumerate(clusters, 1):
clSize = sum(contig2size[c] for c in cluster)
totsize += clSize
contigs += len(cluster)
out.write("\t".join(cluster)+"\n")
logger(" %3s bp in %s clusters generated from %s contigs."%(totsize, len(clusters), contigs))
return clusters
rough_mask_model = get_rough_model(args.weights + 'rough_model_weights.h5',
args.learning_rate, args.decay_rate,
args.no_gpus)
refined_mask_model = get_refined_model(
args.weights + 'refine_model_weights.h5', args.learning_rate,
args.decay_rate, args.no_gpus)
# create the rough masks
mask = rough_mask_model.predict(dataset,
batch_size=args.batch_size * args.no_gpus)
# How many rough POCs?
print("Roughly detected POCs: {}".format(mask.any(axis=(1, 2, 3)).sum()))
# TODO I might be able to speed up the following by only applying it to this subset of images
np.savez_compressed('rough.npz', mask=mask >= 0.5)
# dilate them and apply to the dataset
dataset_masked = get_masked_dataset(mask, dataset)
np.savez_compressed('dilated_mask.npz', mask=dataset_masked >= 0.5)
# refine the masked dataset
refined_mask = refined_mask_model.predict(dataset_masked,
batch_size=args.batch_size *
args.no_gpus)
# Create the final mask
pre = refined_mask >= 0.5
# How many rough POCs?
print("Final detected POCs: {}".format(pre.any(axis=(1, 2, 3)).sum()))
dataset, action_dataset = load_raw_data_list(filelist)
reset_graph()
vae = ConvVAE(z_size=z_size,
batch_size=batch_size,
learning_rate=learning_rate,
kl_tolerance=kl_tolerance,
is_training=False,
reuse=False,
gpu_mode=True) # use GPU on batchsize of 1000 -> much faster
vae.load_json(os.path.join(model_path_name, 'vae.json'))
mu_dataset = []
logvar_dataset = []
for i in range(len(dataset)):
data_batch = dataset[i]
mu, logvar, z = encode_batch(data_batch)
mu_dataset.append(mu.astype(np.float16))
logvar_dataset.append(logvar.astype(np.float16))
if ((i + 1) % 100 == 0):
print(i + 1)
action_dataset = np.array(action_dataset)
mu_dataset = np.array(mu_dataset)
logvar_dataset = np.array(logvar_dataset)
np.savez_compressed(os.path.join(SERIES_DIR, "series.npz"),
action=action_dataset,
mu=mu_dataset,
logvar=logvar_dataset)
def saveMatrix(matrix, outFile):
outf = open(outFile, 'w')
np.savez_compressed(outf, matrix)
outf.close()