def __init__(self):
c = 3.0e+11 # speed of light, um/ms
cs = 2.8e-11 # cross section, um^2
n = 1.5
self.WI = (c/n)*cs # 18.0, um^3/ms
self.Nv = 5.0e+7 # 5.0e+7 1/um^3 - density of active particles
self.W2 = 1./0.23 # 1/ms - 1/spontaneous emission lifetime (2->1 transitions)
self.W3 = 1.e-3*self.W2 # 1/ms - 1/spontaneous emission lifetime (3->2 transitions)
self.eta = 1.e-16
tb, te, self.ts = 0.0, 1.0, 1.0e-4
self.x = arange(tb,te,self.ts)
self.Np = len(self.x)
self.y1 = ndarray(shape=(self.Np), dtype='float')
self.y2 = ndarray(shape=(self.Np), dtype='float')
# self.y3 = ndarray(shape=(self.Np), dtype='float')
self.y4 = ndarray(shape=(self.Np), dtype='float')
self.PlotWindow = Tk.Toplevel()
self.PlotWindow.title('numerical simulation of kinetic equations')
fig = Figure(figsize=(10,6), dpi=100)
self.g1 = fig.add_subplot(211)
self.g2 = fig.add_subplot(212)
self.canvas = FigureCanvasTkAgg(fig, self.PlotWindow)
self.canvas.show()
self.canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)
self.toolbar = NavigationToolbar2TkAgg( self.canvas, self.PlotWindow)
self.canvas._tkcanvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1)
def experienceReplay(self, time):
if self.initial_exploration < time:
# Pick up replay_size number of samples from the Data
if time < self.data_size: # during the first sweep of the History Data
replay_index = np.random.randint(0, time, (self.replay_size, 1))
else:
replay_index = np.random.randint(0, self.data_size, (self.replay_size, 1))
s_replay = np.ndarray(shape=(self.replay_size, 4, 84, 84), dtype=np.float32)
a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8)
r_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.float32)
s_dash_replay = np.ndarray(shape=(self.replay_size, 4, 84, 84), dtype=np.float32)
episode_end_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.bool)
for i in xrange(self.replay_size):
s_replay[i] = np.asarray(self.D[0][replay_index[i]], dtype=np.float32)
a_replay[i] = self.D[1][replay_index[i]]
r_replay[i] = self.D[2][replay_index[i]]
s_dash_replay[i] = np.array(self.D[3][replay_index[i]], dtype=np.float32)
episode_end_replay[i] = self.D[4][replay_index[i]]
s_replay = cuda.to_gpu(s_replay)
s_dash_replay = cuda.to_gpu(s_dash_replay)
# Gradient-based update
self.optimizer.zero_grads()
loss, _ = self.forward(s_replay, a_replay, r_replay, s_dash_replay, episode_end_replay)
loss.backward()
self.optimizer.update()
def make_arrays(nb_rows, img_size):
if nb_rows:
dataset = np.ndarray((nb_rows, img_size, img_size), dtype=np.float32)
labels = np.ndarray(nb_rows, dtype=np.int32)
else:
dataset, labels = None, None
return dataset, labels
def produce_optimization_sets(self, train, test_samples=None):
if test_samples == 0:
return [train, numpy.ndarray([0, 0]), 0]
test_size = int(round(train.shape[0] / 10))
if test_samples is None:
test_samples = random.sample(xrange(0, train.shape[0] - 1), test_size)
train_index = 0
test_index = 0
train_result = numpy.ndarray([train.shape[0] - test_size, train.shape[1]], dtype=theano.config.floatX)
test_result = numpy.ndarray([test_size, train.shape[1]], dtype=theano.config.floatX)
for i in xrange(train.shape[0]):
if i in test_samples:
test_result[test_index, :] = train[i, :]
test_index += 1
else:
train_result[train_index, :] = train[i, :]
train_index += 1
return [train_result, test_result, test_samples]
def generate_performance_results(self, data, configs):
num_folds = len(self.fold_data)
num_pred = len(self.fold_data[0].test_predicted_values)
num_actual = len(self.fold_data[0].test_actual_values)
train_perf = np.ndarray(num_folds)
test_perf = np.ndarray(num_folds)
test_predicted = np.ones((num_folds,num_pred,))
test_actual = np.ones((num_folds,num_actual,))
for index, fold in enumerate(self.fold_data):
# train_targets = self.train_targets[index]
# test_targets = self.test_targets[index]
train_predicted = fold.train_predicted_values
train_actual = fold.train_actual_values
test_predicted[index] = fold.test_predicted_values
test_actual[index] = fold.test_actual_values
results_loss_function = configs.results_loss_function
train_target_ids = data.get_target_ids(fold.train_inds)
test_target_ids = data.get_target_ids(fold.test_inds)
train_perf[index] = LossFunction.compute_loss_function(train_predicted,
train_actual,
train_target_ids,
results_loss_function)
test_perf[index] = LossFunction.compute_loss_function(test_predicted[index],
test_actual[index],
test_target_ids,
results_loss_function)
self.fold_data[index].train_error = train_perf[index]
self.fold_data[index].test_error = test_perf[index]
self.test_actual = test_actual
self.train_actual = train_actual
self.test_predicted = test_predicted
self.train_predicted = train_predicted
def recordDVM(filename='voltdata.npz',sun=False,moon=False,recordLength=np.inf,verbose=True):
ra = 0
dec = 0
raArr = np.ndarray(0)
decArr = np.ndarray(0)
lstArr = np.ndarray(0)
jdArr = np.ndarray(0)
voltArr = np.ndarray(0)
startTime = time.time()
while np.less(time.time()-startTime,recordLength):
if sun:
raDec = sunPos()
ra = raDec[0]
dec = raDec[1]
startSamp = time.time()
currVolt = getDVMData()
currLST = getLST()
currJulDay = getJulDay()
raArr = np.append(raArr,ra)
decArr = np.append(decArr,ra)
voltArr = np.append(voltArr,currVolt)
lstArr = np.append(lstArr,currLST)
jdArr = np.append(jdArr,currJulDay)
if verbose:
print 'Measuring voltage: ' + str(currVolt) + ' (LST: ' + str(currLST) +' ' + time.asctime() + ')'
np.savez(filename,ra=raArr,dec=decArr,jd=jdArr,lst=lstArr,volts=voltArr)
sys.stdout.flush()
time.sleep(np.max([0,1.0-(time.time()-startSamp)]))
def __init__(self, n_in, n_out, weights=None,
activation='sigmoid', is_classifier_layer=False):
# Get activation function from string
self.activation_string = activation
self.activation = Activation.get_activation(self.activation_string)
self.activation_derivative = Activation.get_derivative(
self.activation_string)
self.n_in = n_in
self.n_out = n_out
self.inp = np.ndarray(n_in + 1)
self.inp[0] = 1
self.outp = np.ndarray(n_out)
self.deltas = np.zeros(n_out)
# You can have better initialization here
if weights is None:
self.weights = np.random.rand(n_in + 1, n_out) / 10 - 0.05
# Adjust weights to zero mean
for i in range(n_out):
self.weights[:][i] -= (sum(self.weights[:][i]) / len(self.weights[:][i]))
else:
assert(weights.shape == (n_in + 1, n_out))
self.weights = weights
self.is_classifier_layer = is_classifier_layer
# Some handy properties of the layers
self.size = self.n_out
self.shape = self.weights.shape
def prop_ring(self):
"""
Test properties for a ring, modelled as a thin walled something
"""
radius = 1.
# make sure the simple test cases go well
x = np.linspace(0,radius,100000)
y = np.sqrt(radius*radius - x*x)
x = np.append(-x[::-1], x)
y_up = np.append(y[::-1], y)
tw1 = np.ndarray((len(x),3), order='F')
tw1[:,0] = x
tw1[:,1] = y_up
tw1[:,2] = 0.01
tw2 = np.ndarray((len(x),3), order='F')
y_low = np.append(-y[::-1], -y)
tw2[:,0] = x
tw2[:,1] = y_low
tw2[:,2] = 0.01
# tw1 and tw2 need to be of the same size, give all zeros
upper_bound = sp.zeros((4,2), order='F')
lower_bound = sp.zeros((4,2), order='F')
st_arr, EA, EIxx, EIyy = properties(upper_bound, lower_bound,
tw1=tw1, tw2=tw2, rho=1., rho_tw=1., E=1., E_tw=1.)
headers = HawcPy.ModelData().st_column_header_list
print '\nRING PROPERTIES'
for index, item in enumerate(headers):
tmp = item + ' :'
print tmp.rjust(8), st_arr[index]
def AllocateRAM (self, verbose = True):
ramsz = np.prod(self.dims) * self.ds / MB
if (verbose):
print ' Allocating %.1f MB of RAM' % ramsz
print ' Data (dims: %(a)d %(b)d %(c)d %(d)d %(e)d %(f)d %(g)d %(h)d %(i)d %(j)d %(k)d %(l)d %(m)d %(n)d %(o)d %(p)d)' % {
"a": self.dims[ 0], "b": self.dims[ 1], "c": self.dims[ 2], "d": self.dims[ 3], "e": self.dims[ 4], "f": self.dims[ 5],
"g": self.dims[ 6], "h": self.dims[ 7], "i": self.dims[ 8], "j": self.dims[ 9], "k": self.dims[10], "l": self.dims[11],
"m": self.dims[12], "n": self.dims[13], "o": self.dims[14], "p": self.dims[15]}
print ' Noise (dims: %(a)d %(b)d %(c)d %(d)d %(e)d %(f)d %(g)d %(h)d %(i)d %(j)d %(k)d %(l)d %(m)d %(n)d %(o)d %(p)d)' % {
"a": self.noisedims[ 0], "b": self.noisedims[ 1], "c": self.noisedims[ 2], "d": self.noisedims[ 3], "e": self.noisedims[ 4], "f": self.noisedims[ 5],
"g": self.noisedims[ 6], "h": self.noisedims[ 7], "i": self.noisedims[ 8], "j": self.noisedims[ 9], "k": self.noisedims[10], "l": self.noisedims[11],
"m": self.noisedims[12], "n": self.noisedims[13], "o": self.noisedims[14], "p": self.noisedims[15]}
if (self.syncdims[0]):
print ' SyncData (dims: %(a)d %(b)d )' % {"a": self.syncdims[0], "b": self.syncdims[1]}
self.data = np.ndarray(shape=self.dims, dtype=self.dt)
self.sync = np.ndarray(shape=self.syncdims, dtype=self.sddt)
self.noise = np.ndarray(shape=filter(lambda x:x>0,self.noisedims), dtype=self.nddt)
if (verbose):
print (" ... done.\n" % ramsz)
return
def __init__(self, probs):
prob = numpy.array(probs, numpy.float32)
prob /= numpy.sum(prob)
threshold = numpy.ndarray(len(probs), numpy.float32)
values = numpy.ndarray(len(probs) * 2, numpy.int32)
il, ir = 0, 0
pairs = list(zip(prob, range(len(probs))))
pairs.sort()
for prob, i in pairs:
p = prob * len(probs)
while p > 1 and ir < len(threshold):
values[ir * 2 + 1] = i
p -= 1.0 - threshold[ir]
ir += 1
threshold[il] = p
values[il * 2] = i
il += 1
# fill the rest
for i in range(ir, len(probs)):
values[i * 2 + 1] = 0
assert((values < len(threshold)).all())
self.threshold = threshold
self.values = values
self.use_gpu = False
def get_synthetic_warped_circle(nslices):
#get a subsampled circle
fname_cicle = get_data('reg_o')
circle = np.load(fname_cicle)[::4,::4].astype(floating)
#create a synthetic invertible map and warp the circle
d, dinv = vfu.create_harmonic_fields_2d(64, 64, 0.1, 4)
d = np.asarray(d, dtype=floating)
dinv = np.asarray(dinv, dtype=floating)
mapping = DiffeomorphicMap(2, (64, 64))
mapping.forward, mapping.backward = d, dinv
wcircle = mapping.transform(circle)
if(nslices == 1):
return circle, wcircle
#normalize and form the 3d by piling slices
circle = (circle-circle.min())/(circle.max() - circle.min())
circle_3d = np.ndarray(circle.shape + (nslices,), dtype=floating)
circle_3d[...] = circle[...,None]
circle_3d[...,0] = 0
circle_3d[...,-1] = 0
#do the same with the warped circle
wcircle = (wcircle-wcircle.min())/(wcircle.max() - wcircle.min())
wcircle_3d = np.ndarray(wcircle.shape + (nslices,), dtype=floating)
wcircle_3d[...] = wcircle[...,None]
wcircle_3d[...,0] = 0
wcircle_3d[...,-1] = 0
return circle_3d, wcircle_3d
def polar(self, n, file='None'):
r = np.arange(self.rmin,self.rmax,(self.rmax-self.rmin)/self.nrad)
t = np.arange(0.,2.*np.pi,2.*np.pi/self.nsec)
x = np.ndarray([self.nrad*self.nsec], dtype = float)
y = np.ndarray([self.nrad*self.nsec], dtype = float)
z = np.ndarray([self.nrad*self.nsec], dtype = float)
k = 0
for i in range(self.nrad):
for j in range(self.nsec):
x[k] = r[i]*np.cos(t[j])
y[k] = r[i]*np.sin(t[j])
z[k] = self.data[i,j]
k +=1
xx = np.arange(-self.rmax, self.rmax, (self.rmax-self.rmin)/n)
yy = np.arange(-self.rmax, self.rmax, (self.rmax-self.rmin)/n)
zz = griddata(x,y,z,xx,yy)
fig_pol = plt.figure()
ax1 = fig_pol.add_subplot(111, axisbg='k')
ax1.set_xlabel("X")
ax1.set_ylabel("Y")
if(self.zmax!='None' and self.zmin!='None'):
ax1.imshow(zz, cmap=cm.hot, origin="lower", \
extent=[-self.rmax,self.rmax, \
-self.rmax,self.rmax])
else:
ax1.imshow(zz, cmap=cm.hot, origin="lower", \
extent=[-self.rmax,self.rmax, \
-self.rmax,self.rmax])
if(file!="None"):
plt.savefig(file+".png",dpi=70, format="png" )
print file+".png done"
else:
plt.show()
def dual_plot(self, n):
fig = plt.figure()
ax1 = fig.add_subplot(121, axisbg='k')
ax1.set_xlabel("Theta")
ax1.set_ylabel("R")
ax1.imshow(self.data, cmap=cm.hot,origin="lower", \
extent=[0,2*np.pi,self.rmin,self.rmax])
r = np.arange(self.rmin,self.rmax,(self.rmax-self.rmin)/self.nrad)
t = np.arange(0.,2.*np.pi,2.*np.pi/self.nsec)
x = np.ndarray([self.nrad*self.nsec], dtype = float)
y = np.ndarray([self.nrad*self.nsec], dtype = float)
z = np.ndarray([self.nrad*self.nsec], dtype = float)
k = 0
for i in range(self.nrad):
for j in range(self.nsec):
x[k] = r[i]*np.cos(t[j])
y[k] = r[i]*np.sin(t[j])
z[k] = self.data[i,j]
k +=1
xx = np.arange(-self.rmax, self.rmax, (self.rmax-self.rmin)/n)
yy = np.arange(-self.rmax, self.rmax, (self.rmax-self.rmin)/n)
zz = griddata(x,y,z,xx,yy)
ax2 = fig.add_subplot(122, axisbg='k')
ax2.set_xlabel("X")
ax2.set_ylabel("Y")
ax2.imshow(zz, cmap=cm.hot,origin="lower" \
, extent=[-self.rmax,self.rmax,-self.rmax,self.rmax])
plt.show()
def pd_to_array(inpd, dims=225):
count_vol = numpy.ndarray([dims,dims,dims])
ptids = vtk.vtkIdList()
points = inpd.GetPoints()
data_vol = []
# check for cell data
cell_data = inpd.GetCellData().GetScalars()
if cell_data:
data_vol = numpy.ndarray([dims,dims,dims])
# loop over lines
inpd.GetLines().InitTraversal()
print "<filter.py> Input number of points: ",\
points.GetNumberOfPoints(),\
"lines:", inpd.GetNumberOfLines()
# loop over all lines
for lidx in range(0, inpd.GetNumberOfLines()):
# progress
#if verbose:
# if lidx % 1 == 0:
# print "<filter.py> Line:", lidx, "/", inpd.GetNumberOfLines()
inpd.GetLines().GetNextCell(ptids)
num_points = ptids.GetNumberOfIds()
for pidx in range(0, num_points):
point = points.GetPoint(ptids.GetId(pidx))
# center so that 0,0,0 moves to 100,100,100
point = numpy.round(numpy.array(point) + 110)
count_vol[point[0], point[1], point[2]] += 1
if cell_data:
data_vol[point[0], point[1], point[2]] += cell_data.GetTuple(lidx)[0]
return count_vol, data_vol
def load(data_folders, min_num_images, max_num_images):
dataset = np.ndarray(
shape=(max_num_images, image_size, image_size), dtype=np.float32)
labels = np.ndarray(shape=(max_num_images), dtype=np.int32)
label_index = 0
image_index = 0
for folder in data_folders:
print(folder)
for image in os.listdir(folder):
if image_index >= max_num_images:
raise Exception('More images than expected: %d >= %d' % (
num_images, max_num_images))
image_file = os.path.join(folder, image)
try:
image_data = (ndimage.imread(image_file).astype(float) -
pixel_depth / 2) / pixel_depth
if image_data.shape != (image_size, image_size):
raise Exception('Unexpected image shape: %s' % str(image_data.shape))
dataset[image_index, :, :] = image_data
labels[image_index] = label_index
image_index += 1
except IOError as e:
print('Could not read:', image_file, ':', e, '- it\'s ok, skipping.')
label_index += 1
num_images = image_index
dataset = dataset[0:num_images, :, :]
labels = labels[0:num_images]
if num_images < min_num_images:
raise Exception('Many fewer images than expected: %d < %d' % (
num_images, min_num_images))
print('Full dataset tensor:', dataset.shape)
print('Mean:', np.mean(dataset))
print('Standard deviation:', np.std(dataset))
print('Labels:', labels.shape)
return dataset, labels
def initial_buffers(num_particles):
np_position = numpy.ndarray((num_particles, 4), dtype=numpy.float32)
np_color = numpy.ndarray((num_particles, 4), dtype=numpy.float32)
np_velocity = numpy.ndarray((num_particles, 4), dtype=numpy.float32)
np_position[:,0] = numpy.sin(numpy.arange(0., num_particles) * 2.001 * numpy.pi / num_particles)
np_position[:,0] *= numpy.random.random_sample((num_particles,)) / 3. + .2
np_position[:,1] = numpy.cos(numpy.arange(0., num_particles) * 2.001 * numpy.pi / num_particles)
np_position[:,1] *= numpy.random.random_sample((num_particles,)) / 3. + .2
np_position[:,2] = 0.
np_position[:,3] = 1.
np_color[:,:] = [1.,1.,1.,1.] # White particles
np_velocity[:,0] = np_position[:,0] * 2.
np_velocity[:,1] = np_position[:,1] * 2.
np_velocity[:,2] = 3.
np_velocity[:,3] = numpy.random.random_sample((num_particles, ))
gl_position = vbo.VBO(data=np_position, usage=GL_DYNAMIC_DRAW, target=GL_ARRAY_BUFFER)
gl_position.bind()
gl_color = vbo.VBO(data=np_color, usage=GL_DYNAMIC_DRAW, target=GL_ARRAY_BUFFER)
gl_color.bind()
return (np_position, np_velocity, gl_position, gl_color)
def ch(X, cIDX, distance="euclidean"):
Nclusters = cIDX.max() + 1
Npoints = len(X)
n = np.ndarray(shape=(Nclusters), dtype=float)
j = 0
for i in range(cIDX.min(), cIDX.max() + 1):
aux = np.asarray([float(b) for b in (cIDX == i)])
n[j] = aux.sum()
j = j + 1
# Clusters
A = np.array([X[np.where(cIDX == i)] for i in range(Nclusters)])
# Centroids
v = np.array([np.sum(Ai, axis=0) / float(Ai.shape[0]) for Ai in A])
ssb = 0
for i in range(Nclusters):
ssb = n[i] * (cdist([v[i]], [np.mean(X, axis=0)], metric=distance)[0][0] ** 2) + ssb
z = np.ndarray(shape=(Nclusters), dtype=float)
for i in range(cIDX.min(), cIDX.max() + 1):
aux = np.array([(cdist([x], [v[i]], metric=distance)[0][0] ** 2) for x in X[cIDX == i]])
z[i] = aux.sum()
ssw = z.sum()
return (ssb / (Nclusters - 1)) / (ssw / (Npoints - Nclusters))
def create_test_data():
train_data_path = os.path.join(data_path, 'test')
images = os.listdir(train_data_path)
total = len(images)
imgs = np.ndarray((total, image_rows, image_cols), dtype=np.uint8)
imgs_id = np.ndarray((total, ), dtype=np.int32)
i = 0
print('-'*30)
print('Creating test images...')
print('-'*30)
for image_name in images:
img_id = int(image_name.split('.')[0])
img = imread(os.path.join(train_data_path, image_name), as_grey=True)
img = np.array([img])
imgs[i] = img
imgs_id[i] = img_id
if i % 100 == 0:
print('Done: {0}/{1} images'.format(i, total))
i += 1
print('Loading done.')
np.save('imgs_test.npy', imgs)
np.save('imgs_id_test.npy', imgs_id)
print('Saving to .npy files done.')
def __init__(self, file_name):
# print 'construct skel'
self.vertices = np.ndarray((0,4))
self.projected_vertices = np.ndarray((0,3))
self.lines = np.ndarray((0,4))
self.matrix = np.ndarray((4,4))
file = open(file_name, "r")
file.close
lines = file.readlines()
arr = []
row = 0
v_num = 0
p_num = 0
# for i in range(0, len(lines)):
for line in lines:
line = line.replace("\n", "")
_arr = line.split(" ")
if _arr.__contains__(''):
_arr.remove('')
if _arr[0].__contains__("#"):
# print 'skip'
continue
row += 1
if row == 1:
type = _arr[0]
if row == 2:
v_num = int(_arr[0])
p_num = int(_arr[1])
if row > 2 and row <= 2 + v_num:
_arr = map(int, _arr)
self.vertices = np.concatenate((self.vertices, [_arr]), axis=0)
if row > 2+v_num and row <= 2+v_num+p_num:
_arr = map(int, _arr)
self.lines = np.concatenate((self.lines, [_arr]), axis=0)
def create_small_train_data(self):
# 将增强之后的训练集生成npy
print('-' * 30)
print('creating samll train image')
print('-' * 30)
imgs = glob.glob('../data_set/aug_train/0/*' + '.tif')
count = len(imgs)
imgdatas = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
imglabels = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
trainPath = '../data_set/aug_train/0'
labelPath = '../data_set/aug_label/0'
i = 0
for imgname in imgs:
trainmidname = imgname[imgname.rindex('/') + 1:]
labelimgname = imgname[imgname.rindex('/') + 1:imgname.rindex('_')] + '_label.tif'
print(trainmidname, labelimgname)
img = load_img(trainPath + '/' + trainmidname, grayscale=True)
label = load_img(labelPath + '/' + labelimgname, grayscale=True)
img = img_to_array(img)
label = img_to_array(label)
imgdatas[i] = img
imglabels[i] = label
i += 1
print(i)
print('loading done', imgdatas.shape)
np.save(self.npy_path + '/imgs_small_train.npy', imgdatas) # 将30张训练集和30张label生成npy数据
np.save(self.npy_path + '/imgs_mask_small_train.npy', imglabels)
print('Saving to .npy files done.')
def create_train_data():
train_data_path = os.path.join(data_path, 'train')
images = os.listdir(train_data_path)
total = int(len(images) / 2)
imgs = np.ndarray((total, image_rows, image_cols), dtype=np.uint8)
imgs_mask = np.ndarray((total, image_rows, image_cols), dtype=np.uint8)
i = 0
print('-'*30)
print('Creating training images...')
print('-'*30)
for image_name in images:
if 'mask' in image_name:
continue
image_mask_name = image_name.split('.')[0] + '_mask.tif'
img = imread(os.path.join(train_data_path, image_name), as_grey=True)
img_mask = imread(os.path.join(train_data_path, image_mask_name), as_grey=True)
img = np.array([img])
img_mask = np.array([img_mask])
imgs[i] = img
imgs_mask[i] = img_mask
if i % 100 == 0:
print('Done: {0}/{1} images'.format(i, total))
i += 1
print('Loading done.')
np.save('imgs_train.npy', imgs)
np.save('imgs_mask_train.npy', imgs_mask)
print('Saving to .npy files done.')
def extract(self, image, segments):
fs = self.feature_size
bg = 255
regions = numpy.ndarray(shape=(0, fs), dtype=FEATURE_DATATYPE)
for segment in segments:
region = region_from_segment(image, segment)
if self.stretch:
region = cv2.resize(region, (fs, fs))
else:
x, y, w, h = segment
proportion = float(min(h, w)) / max(w, h)
new_size = (fs, int(fs * proportion)) if min(w, h) == h else (int(fs * proportion), fs)
region = cv2.resize(region, new_size)
s = region.shape
new_region = numpy.ndarray((fs, fs), dtype=region.dtype)
new_region[:, :] = bg
new_region[:s[0], :s[1]] = region
region = new_region
regions = numpy.append(regions, region, axis=0)
regions.shape = (len(segments), fs**2)
return regions
def get_label_voxels(self):
#the voxel coordinates of fg and bg labels
if not self.opLabelArray.NonzeroBlocks.ready():
return (None,None)
nonzeroSlicings = self.opLabelArray.NonzeroBlocks[:].wait()[0]
coors1 = [[], [], []]
coors2 = [[], [], []]
for sl in nonzeroSlicings:
a = self.opLabelArray.Output[sl].wait()
w1 = numpy.where(a == 1)
w2 = numpy.where(a == 2)
w1 = [w1[i] + sl[i].start for i in range(1,4)]
w2 = [w2[i] + sl[i].start for i in range(1,4)]
for i in range(3):
coors1[i].append( w1[i] )
coors2[i].append( w2[i] )
for i in range(3):
if len(coors1[i]) > 0:
coors1[i] = numpy.concatenate(coors1[i],0)
else:
coors1[i] = numpy.ndarray((0,), numpy.int32)
if len(coors2[i]) > 0:
coors2[i] = numpy.concatenate(coors2[i],0)
else:
coors2[i] = numpy.ndarray((0,), numpy.int32)
return (coors2, coors1)
def generate_batch(config, data, unique_neg_data):
global batch_idx, data_idx
batch_size = config['batch_size']
neg_sample_size = config['neg_sample_size']
batch = data.values()[batch_idx]
neg_batch = unique_neg_data.values()[batch_idx]
idx = data_idx[batch_idx]
data_pos_x = np.ones(batch_size) * batch_idx
data_pos_y = np.ndarray(shape=batch_size, dtype=np.int32)
data_neg_y = np.ndarray(shape=neg_sample_size, dtype=np.int32)
for i in xrange(batch_size):
data_pos_y[i] = batch[idx]
idx = (idx + 1) % len(batch)
for i, neg_y_idx in enumerate(random.sample(set(neg_batch), neg_sample_size)):
data_neg_y[i] = neg_y_idx
data_idx[batch_idx] = idx
batch_idx = (batch_idx + 1) % len(data)
return data_pos_x, data_pos_y, data_neg_y
def load(data_folders, min_num_images, max_num_images):
dataset = np.ndarray(
shape=(max_num_images, image_size, image_size), dtype=np.float32)
labels = np.ndarray(shape=(max_num_images), dtype=np.int32)
label_index = 0
image_index = 0
for folder in data_folders:
print folder
for image in os.listdir(folder):
if image_index >= max_num_images:
raise Exception('More images than expected: %d > %d'
(num_images, max_num_images))
image_file = os.path.join(folder, image)
try:
# Loads the file. There seems to be some sort of translation on it.
image_data = (ndimage.imread(image_file).astype(float) - pixel_depth / 2) / pixel_depth
if image_data.shape != (image_size, image_size):
raise Exception('Unexpected image shape: %s' % str(image_data.shape))
dataset[image_index, :, :] = image_data
labels[image_index] = label_index
image_index += 1
except IOError as e:
print 'Could not read: ', image_file, ': ', e, '- skipped.'
label_index += 1
num_images = image_index
dataset = dataset[0:num_images, :, :]
labels = labels[0:num_images]
if num_images < min_num_images:
raise Exception('Many fewer images than expected: %d < %d' %
(num_images, min_num_images))
print 'Full dataset tensor: ', dataset.shape
print 'Mean: ', np.mean(dataset)
print 'Stdev: ', np.std(dataset)
print 'Labels: ', labels.shape
return dataset, labels
def contract_k(pLqR, pLqI):
# K ~ 'iLj,lLk*,li->kj' + 'lLk*,iLj,li->kj'
#:pLq = (LpqR + LpqI.reshape(-1,nao,nao)*1j).transpose(1,0,2)
#:tmp = numpy.dot(dm, pLq.reshape(nao,-1))
#:vk += numpy.dot(pLq.reshape(-1,nao).conj().T, tmp.reshape(-1,nao))
nrow = pLqR.shape[1]
tmpR = numpy.ndarray((nao, nrow * nao), buffer=buf2R)
if k_real:
for i in range(nset):
lib.ddot(dmsR[i], pLqR.reshape(nao, -1), 1, tmpR)
lib.ddot(pLqR.reshape(-1, nao).T, tmpR.reshape(-1, nao), 1, vkR[i], 1)
else:
tmpI = numpy.ndarray((nao, nrow * nao), buffer=buf2I)
for i in range(nset):
zdotNN(dmsR[i], dmsI[i], pLqR.reshape(nao, -1), pLqI.reshape(nao, -1), 1, tmpR, tmpI, 0)
zdotCN(
pLqR.reshape(-1, nao).T,
pLqI.reshape(-1, nao).T,
tmpR.reshape(-1, nao),
tmpI.reshape(-1, nao),
1,
vkR[i],
vkI[i],
1,
)
def cv(xs,ys): errorsums = np.zeros(11) bestdeg = 0 def sqerror(y, yhat): return (y-yhat)**2 segment = len(xs)/10 for d in xrange(0,11): #each K for i in xrange(0,10): #each segment trp = 0 #points next position in train arrays tep = 0 #points next position in test arrays tsi = segment*i #Testdata start index xtrain = np.ndarray(len(xs)-segment) ytrain = np.ndarray(len(xs)-segment) xtest = np.ndarray(segment) ytest = np.ndarray(segment) for j in xrange(0,len(xs)): #divide data if j<tsi or j>=tsi+segment: xtrain[trp] = xs[j] ytrain[trp] = ys[j] trp+=1 else: xtest[tep] = xs[j] ytest[tep] = ys[j] tep+=1 polynomial = fitPolynomial(d, xtrain, ytrain) for k in xrange(0, len(ytest)): errorsums[d] += sqerror(ytest[k], polynomial(xtest[k])) if errorsums[d] < errorsums[bestdeg]: bestdeg = d return [bestdeg,errorsums]
def shape_from_header(self, hdr):
'''Read the shape of the array described by the header.
The file position after this call is unspecified.
'''
mclass = hdr.mclass
if mclass == mxFULL_CLASS:
shape = tuple(map(int, hdr.dims))
elif mclass == mxCHAR_CLASS:
shape = tuple(map(int, hdr.dims))
if self.chars_as_strings:
shape = shape[:-1]
elif mclass == mxSPARSE_CLASS:
dt = hdr.dtype
dims = hdr.dims
if not (len(dims) == 2 and dims[0] >= 1 and dims[1] >= 1):
return ()
# Read only the row and column counts
self.mat_stream.seek(dt.itemsize * (dims[0] - 1), 1)
rows = np.ndarray(shape=(1,), dtype=dt,
buffer=self.mat_stream.read(dt.itemsize))
self.mat_stream.seek(dt.itemsize * (dims[0] - 1), 1)
cols = np.ndarray(shape=(1,), dtype=dt,
buffer=self.mat_stream.read(dt.itemsize))
shape = (int(rows), int(cols))
else:
raise TypeError('No reader for class code %s' % mclass)
if self.squeeze_me:
shape = tuple([x for x in shape if x != 1])
return shape
def load_gl_buffers(self):
num = self.n_frags
pos = np.ndarray((num, 4), dtype=np.float32)
seed = np.random.rand(2,num)
pos[:,0] = seed[0,:]
pos[:,1] = 0.0
pos[:,2] = seed[1,:] # z pos
pos[:,3] = 1. # velocity
# pos[:,1] = np.sin(np.arange(0., num) * 2.001 * np.pi / (10*num))
# pos[:,1] *= np.random.random_sample((num,)) / 3. - 0.2
# pos[:,2] = np.cos(np.arange(0., num) * 2.001 * np.pi /(10* num))
# pos[:,2] *= np.random.random_sample((num,)) / 3. - 0.2
# pos[:,0] = 0. # z pos
# pos[:,3] = 1. # velocity
self.pos = pos
self.pos_vbo = vbo.VBO(data=self.pos, usage=GL_DYNAMIC_DRAW, target=GL_ARRAY_BUFFER)
# self.pos_vbo = vbo.VBO(data=self.pos_vect_frags_4_GL, usage=GL_DYNAMIC_DRAW, target=GL_ARRAY_BUFFER)
self.pos_vbo.bind()
self.col_vbo = vbo.VBO(data=self.col_vect_frags_4_GL, usage=GL_DYNAMIC_DRAW, target=GL_ARRAY_BUFFER)
self.col_vbo.bind()
self.vel = np.ndarray((self.n_frags, 4), dtype=np.float32)
self.vel[:,2] = self.pos[:,2] * 2.
self.vel[:,1] = self.pos[:,1] * 2.
self.vel[:,0] = 3.
self.vel[:,3] = np.random.random_sample((self.n_frags, ))
def create_train_data(self):
# 将增强之后的训练集生成npy
i = 0
print('-' * 30)
print('creating train image')
print('-' * 30)
count = 0
for indir in os.listdir(self.aug_merge_path):
path = os.path.join(self.aug_merge_path, indir)
count += len(os.listdir(path))
imgdatas = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
imglabels = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
for indir in os.listdir(self.aug_merge_path):
trainPath = os.path.join(self.aug_train_path, indir)
labelPath = os.path.join(self.aug_label_path, indir)
print(trainPath, labelPath)
imgs = glob.glob(trainPath + '/*' + '.tif')
for imgname in imgs:
trainmidname = imgname[imgname.rindex('/') + 1:]
labelimgname = imgname[imgname.rindex('/') + 1:imgname.rindex('_')] + '_label.tif'
print(trainmidname, labelimgname)
img = load_img(trainPath + '/' + trainmidname, grayscale=True)
label = load_img(labelPath + '/' + labelimgname, grayscale=True)
img = img_to_array(img)
label = img_to_array(label)
imgdatas[i] = img
imglabels[i] = label
if i % 100 == 0:
print('Done: {0}/{1} images'.format(i, len(imgs)))
i += 1
print(i)
print('loading done', imgdatas.shape)
np.save(self.npy_path + '/imgs_train.npy', imgdatas) # 将30张训练集和30张label生成npy数据
np.save(self.npy_path + '/imgs_mask_train.npy', imglabels)
print('Saving to .npy files done.')
# Beta distribution CDF when 5*alpha = beta.
# Idea from Dr. J. Rodal
# Use Inkscape to enlarge and adjust the figure position
# (degroup, enlarge and regroup the SVG elements etc.).
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as pl
from matplotlib import cm
import numpy as np
from scipy.special import beta, betainc
import scipy.integrate as integral
# Prepare the data by Ix(a, a)
X = np.arange(0.0, 1.0, .02)
A = np.arange(0.0, 10.0, .002)
gridX, gridA = np.meshgrid(X, A)
Z = np.ndarray(shape=gridX.shape, dtype=float)
for i in range(len(X)):
for j in range(len(A)):
if A[j] < 0.001:
Z[j][i] = betainc(0.001, 0.005, X[i])
else:
Z[j][i] = betainc(A[j], 5 * A[j], X[i])
# Draw the data
fig = pl.figure()
ax = fig.add_subplot(111, projection='3d')
pl.xticks([0.0, 0.5, 1.0])
pl.yticks([0.0, 5.0, 10.0])
ax.contour(gridX, gridA, Z, zdir='z', offset=0)
ax.plot_surface(gridX, gridA, Z, lw=0.0)
ax.plot_wireframe(gridX, gridA, Z, lw = 1.0, color = 'black', \
rstride=1000, cstride=10)
def make_simple_trace(bbfile='grism.fits',outname='grismtrace',ybox=None,xbox=None,noisemaxfact=0.05,alph=1.0,Q=1.0,rotate=False,resize=None):
go=pyfits.open(bbfile)
redshift=go['BROADBAND'].header['REDSHIFT']
wfc3_pix_as=0.13
g141_nm_per_pix=4.65
min_lam=1.075
max_lam=1.700
hdu=go['CAMERA0-BROADBAND-NONSCATTER']
cube=hdu.data #L_lambda units!
#cube=np.flipud(cube) ; print(cube.shape)
fil=go['FILTERS']
lamb=fil.data['lambda_eff']*1.0e6
flux=fil.data['L_lambda_eff_nonscatter0']
g141_i = (lamb >= min_lam) & (lamb <= max_lam)
arcsec_per_kpc= gsu.illcos.arcsec_per_kpc_proper(redshift)
kpc_per_arcsec=1.0/arcsec_per_kpc.value
im_kpc=hdu.header['CD1_1']
print('pix size kpc: ', im_kpc)
wfc3_kpc_per_pix=wfc3_pix_as*kpc_per_arcsec
total_width_pix=(1.0e3)*(max_lam-min_lam)/g141_nm_per_pix
total_width_kpc=total_width_pix*wfc3_kpc_per_pix
total_width_impix=int(total_width_kpc/im_kpc)
delta_lam=(max_lam-min_lam)/total_width_impix #microns/pix
psf_arcsec=0.18
psf_kpc=psf_arcsec*kpc_per_arcsec
psf_impix=psf_kpc/im_kpc
imw_cross=200
imw_disp=total_width_impix+imw_cross
Np=cube.shape[-1]
mid = np.int64(Np/2)
delt=np.int64(imw_cross/2)
output_image=np.zeros_like( np.ndarray(shape=(imw_disp,imw_cross),dtype='float' ))
#r = r[mid-delt:mid+delt,mid-delt:mid+delt]
output_image.shape
small_cube=cube[g141_i,mid-delt:mid+delt,mid-delt:mid+delt]
for i,l in enumerate(lamb[g141_i]):
di=int( (l-min_lam)/delta_lam )
this_cube=small_cube[i,:,:]*l**2 #convert to Janskies-like
if rotate is True:
this_cube = np.rot90(this_cube)
#if i==17:
# this_cube[30,30] = 1.0e3
#print(i,l/(1.0+redshift),int(di),np.sum(this_cube),this_cube.shape,output_image.shape,output_image[di:di+imw_cross,:].shape)
output_image[di:di+imw_cross,:]=output_image[di:di+imw_cross,:]+this_cube
output_image=scipy.ndimage.gaussian_filter(output_image,sigma=[4,psf_impix/2.355])
new_thing = np.transpose(np.flipud(output_image))
if resize is not None:
new_thing = congrid.congrid(new_thing, resize)
nr = noisemaxfact*np.max(new_thing)*random.randn(new_thing.shape[0],new_thing.shape[1])
#thing=make_color_image.make_interactive(new_thing+nr,new_thing+nr,new_thing+nr,alph=alph,Q=Q)
#thing=1.0-np.fliplr(np.transpose(thing,axes=[1,0,2]))
thing=np.fliplr(new_thing+nr)
f=plt.figure(figsize=(25,6))
f.subplots_adjust(wspace=0.0,hspace=0.0,top=0.99,right=0.99,left=0,bottom=0)
axi=f.add_subplot(1,1,1)
axi.imshow( (thing),aspect='auto',origin='left',interpolation='nearest',cmap='Greys_r')
f.savefig(outname+'.png',dpi=500)
plt.close(f)
#[ybox[0]:ybox[1],xbox[0]:xbox[1]]
#[50:125,120:820,:]
new_hdu=pyfits.PrimaryHDU(thing)
new_list=pyfits.HDUList([new_hdu])
new_list.writeto(outname+'.fits',clobber=True)
return thing, new_thing
from DBN_Gaussian_timit import DBN # not Gaussian if no GRBM
with open(sys.argv[3]) as idbnf:
dbn = cPickle.load(idbnf)
with open(sys.argv[4]) as idbndtf:
dbn_to_int_to_state_tuple = cPickle.load(idbndtf)
dbn_phones_to_states = dbn_to_int_to_state_tuple[0]
likelihoods_computer = functools.partial(compute_likelihoods_dbn, dbn)
# TODO bigrams
transitions = initialize_transitions(transitions)
#print transitions
transitions = penalty_scale(transitions,
insertion_penalty=INSERTION_PENALTY,
scale_factor=SCALE_FACTOR)
dummy = np.ndarray((2, 2)) # to force only 1 compile of Viterbi's C
viterbi(dummy, [None, dummy], {}) # also for this compile's debug purposes
list_of_mfcc_files = []
for d, ds, fs in os.walk(sys.argv[1]):
for fname in fs:
if fname[-4:] != '.mfc':
continue
fullname = d.rstrip('/') + '/' + fname
list_of_mfcc_files.append(fullname)
#print list_of_mfcc_files
if dbn != None:
input_n_frames = dbn.rbm_layers[0].n_visible / 39 # TODO generalize
print "this is a DBN with", input_n_frames, "frames on the input layer"
print "concatenating MFCC files"
def featurize_trackers(
self,
trackers: List[DialogueStateTracker],
domain: Domain,
precomputations: Optional[MessageContainerForCoreFeaturization],
bilou_tagging: bool = False,
ignore_action_unlikely_intent: bool = False,
) -> Tuple[
List[List[Dict[Text, List[Features]]]],
np.ndarray,
List[List[Dict[Text, List[Features]]]],
]:
"""Featurizes the training trackers.
Args:
trackers: list of training trackers
domain: the domain
precomputations: Contains precomputed features and attributes.
bilou_tagging: indicates whether BILOU tagging should be used or not
ignore_action_unlikely_intent: Whether to remove `action_unlikely_intent`
from training state features.
Returns:
- a dictionary of state types (INTENT, TEXT, ACTION_NAME, ACTION_TEXT,
ENTITIES, SLOTS, ACTIVE_LOOP) to a list of features for all dialogue
turns in all training trackers
- the label ids (e.g. action ids) for every dialogue turn in all training
trackers
- A dictionary of entity type (ENTITY_TAGS) to a list of features
containing entity tag ids for text user inputs otherwise empty dict
for all dialogue turns in all training trackers
"""
self.prepare_for_featurization(domain, bilou_tagging)
(
trackers_as_states,
trackers_as_labels,
trackers_as_entities,
) = self.training_states_labels_and_entities(
trackers,
domain,
ignore_action_unlikely_intent=ignore_action_unlikely_intent,
)
tracker_state_features = self._featurize_states(
trackers_as_states, precomputations
)
if not tracker_state_features and not trackers_as_labels:
# If input and output were empty, it means there is
# no data on which the policy can be trained
# hence return them as it is. They'll be handled
# appropriately inside the policy.
return tracker_state_features, np.ndarray(trackers_as_labels), []
label_ids = self._convert_labels_to_ids(trackers_as_labels, domain)
entity_tags = self._create_entity_tags(
trackers_as_entities, precomputations, bilou_tagging
)
return tracker_state_features, label_ids, entity_tags
def write_candidates(output_dir,
catId, tract, patch, objId, nVisit, pfsVisitHash,
lambda_ranges, mask, candidates, models, zpdf, linemeas, object_class):
"""Create a pfsZcandidates FITS file from an amazed output directory."""
path = "pfsZcandidates-%03d-%05d-%s-%016x-%03d-0x%016x.fits" % (
catId, tract, patch, objId, nVisit % 1000, pfsVisitHash)
print("Saving {} redshifts to {}".format(len(candidates),
os.path.join(output_dir, path)))
header = [fits.Card('tract', tract, 'Area of the sky'),
fits.Card('patch', patch, 'Region within tract'),
fits.Card('catId', catId, 'Source of the objId'),
fits.Card('objId', objId, 'Unique ID for object'),
fits.Card('nvisit', nVisit, 'Number of visit'),
fits.Card('vHash', pfsVisitHash, '63-bit SHA-1 list of visits')]
hdr = fits.Header(header)
primary = fits.PrimaryHDU(header=hdr)
hdul = [primary]
if object_class == 'GALAXY':
npix = len(lambda_ranges)
# data['PDU'] = np.array([])
# create ZCANDIDATES HDU
zcandidates = np.ndarray((len(candidates),),
dtype=[('Z', 'f8'), ('Z_ERR', 'f8'),
('ZRANK', 'i4'),
('RELIABILITY', 'f8'),
('CLASS', 'S15'),
('SUBCLASS', 'S15'),
('MODELFLUX', 'f8', (npix,))])
for i, candidate in enumerate(candidates):
zcandidates[i]['Z'] = candidate.redshift
zcandidates[i]['Z_ERR'] = candidate.deltaz
zcandidates[i]['ZRANK'] = candidate.rank
zcandidates[i]['RELIABILITY'] = candidate.intgProba
zcandidates[i]['CLASS'] = object_class
zcandidates[i]['SUBCLASS'] = ''
model = np.array(lambda_ranges, dtype=np.float64, copy=True)
model.fill(np.nan)
np.place(model, mask == 0, models[i])
zcandidates[i]['MODELFLUX'] = np.array(model)
hdul.append(fits.BinTableHDU(name='ZCANDIDATES', data=zcandidates))
# create LAMBDA_SCALE HDU
lambda_scale = np.array(lambda_ranges, dtype=[('WAVELENGTH', 'f4')])
hdul.append(fits.BinTableHDU(name='MODELWL', data=lambda_scale))
# create ZPDF HDU
zpdf_hdu = np.ndarray(len(zpdf), buffer=zpdf,
dtype=[('REDSHIFT', 'f8'), ('PDF', 'f8')])
hdul.append(fits.BinTableHDU(name='ZPDF', data=zpdf_hdu))
# create ZLINES HDU
if linemeas is not None :
zlines = np.ndarray((len(linemeas),),
dtype=[('LINENAME', 'S15'),
('LINEWAVE', 'f8'),
('LINEZ', 'f8'),
('LINEZ_ERR', 'f8'),
('LINESIGMA', 'f8'),
('LINESIGMA_ERR', 'f8'),
('LINEVEL', 'f8'),
('LINEVEL_ERR', 'f8'),
('LINEFLUX', 'f8'),
('LINEFLUX_ERR', 'f8'),
('LINEEW', 'f8'),
('LINEEW_ERR', 'f8'),
('LINECONTLEVEL', 'f8'),
('LINECONTLEVEL_ERR', 'f8')])
for i, lm in enumerate(linemeas):
zlines[i]['LINENAME'] = lm.name
zlines[i]['LINEWAVE'] = lm.lambda_obs # TODO: or lambda_rest_beforeOffset ?
zlines[i]['LINEZ'] = np.nan # TODO: what is that ?
zlines[i]['LINEZ_ERR'] = np.nan # TODO: what is that ?
zlines[i]['LINESIGMA'] = lm.sigma
zlines[i]['LINESIGMA_ERR'] = np.nan # TODO: what is that ?
zlines[i]['LINEVEL'] = lm.velocity
zlines[i]['LINEVEL_ERR'] = np.nan # TODO: what is that
zlines[i]['LINEFLUX'] = lm.flux
zlines[i]['LINEFLUX_ERR'] = lm.flux_err
zlines[i]['LINEEW'] = np.nan # TODO: what is that
zlines[i]['LINEEW_ERR'] = np.nan # TODO: what is that
zlines[i]['LINECONTLEVEL'] = np.nan # TODO: what is that
zlines[i]['LINECONTLEVEL_ERR'] = np.nan # TODO: what is that
hdul.append(fits.BinTableHDU(name='ZLINES', data=zlines))
elif object_class == 'STAR':
# create ZCANDIDATES HDU
zcandidates = np.ndarray((len(candidates),),
dtype=[('Z', 'f8'), ('Z_ERR', 'f8'),
('ZRANK', 'i4'),
('RELIABILITY', 'f8'),
('CLASS', 'S15'),
('SUBCLASS', 'S15')])
for i, candidate in enumerate(candidates):
zcandidates[i]['Z'] = candidate.redshift
zcandidates[i]['Z_ERR'] = 0.
zcandidates[i]['ZRANK'] = 0
zcandidates[i]['RELIABILITY'] = candidate.intgProba
zcandidates[i]['CLASS'] = object_class
zcandidates[i]['SUBCLASS'] = candidate.template
hdul.append(fits.BinTableHDU(name='ZCANDIDATES', data=zcandidates))
fits.HDUList(hdul).writeto(os.path.join(output_dir, path),
overwrite=True)
return path
def get_batch(self, split, batch_size=None, seq_per_img=None):
split_ix = self.split_ix[split]
batch_size = batch_size or self.batch_size
seq_per_img = seq_per_img or self.seq_per_img
fc_batch = np.ndarray(
(batch_size,
self.fc_feat_size), dtype='float32') if self.pre_ft else None
att_batch = np.ndarray(
(batch_size, 14, 14, self.att_feat_size),
dtype='float32') if self.pre_ft and self.att_im else None
img_batch = np.ndarray([batch_size, 3, 224, 224],
dtype='float32') if not (self.pre_ft) else None
label_batch = np.zeros(
[batch_size * self.seq_per_img, self.seq_length + 2], dtype='int')
mask_batch = np.zeros(
[batch_size * self.seq_per_img, self.seq_length + 2],
dtype='float32')
attrs_batch = np.zeros(
[batch_size, self.top_attrs],
dtype='int') if self.attrs_in or self.attrs_out else None
attrs_prob_batch = np.zeros(
[batch_size, self.top_attrs],
dtype='float32') if self.attrs_in or self.attrs_out else None
max_index = len(split_ix)
wrapped = False
infos = []
gts = []
for i in range(batch_size):
import time
t_start = time.time()
ri = self.iterators[split]
ri_next = ri + 1
if ri_next >= max_index:
ri_next = 0
wrapped = True
self.iterators[split] = ri_next
ix = split_ix[ri]
# fetch image
if self.pre_ft: fc_batch[i] = self.h5_fc_file['fc'][ix, :]
if self.pre_ft and self.att_im:
att_batch[i] = self.h5_att_file['att'][ix, :, :, :]
if not (self.pre_ft):
#img = self.load_image(self.image_info[ix]['filename'])
img = self.h5_im_file['images'][ix, :, :, :]
if self.cnn_model == 'resnet101':
img_batch[i] = preprocess(
torch.from_numpy(
img[:, 16:-16, 16:-16].astype('float32') /
255.0)).numpy()
else:
img_batch[i] = preprocess_vgg16(
torch.from_numpy(
img[:, 16:-16, 16:-16].astype('float32'))).numpy()
# fetch the semantic_attributes
if self.attrs_in or self.attrs_out:
if len(self.opt.input_attrs_h5) > 0:
attrs_batch[i] = self.h5_attrs_file['pred_semantic_words'][
ix, :self.top_attrs]
attrs_prob_batch[i] = self.h5_attrs_file[
'pred_semantic_words_prob'][ix, :self.top_attrs]
else:
attrs_batch[i] = self.h5_label_file['semantic_words'][
ix, :self.top_attrs]
# fetch the sequence labels
ix1 = self.label_start_ix[ix] - 1 #label_start_ix starts from 1
ix2 = self.label_end_ix[ix] - 1
ncap = ix2 - ix1 + 1 # number of captions available for this image
assert ncap > 0, 'an image does not have any label. this can be handled but right now isn\'t'
if ncap < self.seq_per_img:
# we need to subsample (with replacement)
seq = np.zeros([self.seq_per_img, self.seq_length],
dtype='int')
for q in range(self.seq_per_img):
ixl = random.randint(ix1, ix2)
seq[q, :] = self.h5_label_file['labels'][
ixl, :self.seq_length]
else:
ixl = random.randint(ix1, ix2 - self.seq_per_img + 1)
seq = self.h5_label_file['labels'][
ixl:ixl + self.seq_per_img, :self.seq_length]
label_batch[i * self.seq_per_img:(i + 1) * self.seq_per_img,
1:self.seq_length + 1] = seq
# Used for reward evaluation
gts_labels = self.h5_label_file['labels'][self.label_start_ix[ix] -
1:self.label_end_ix[ix]]
gts.append(gts_labels)
# record associated info as well
info_dict = {}
info_dict['ix'] = ix
info_dict['id'] = self.info['images'][ix]['id']
info_dict['file_path'] = self.info['images'][ix]['file_path']
infos.append(info_dict)
# generate mask
t_start = time.time()
nonzeros = np.array(map(lambda x: (x != 0).sum() + 2, label_batch))
for ix, row in enumerate(mask_batch):
row[:nonzeros[ix]] = 1
data = {}
data['fc_feats'] = fc_batch # if pre_ft is 0, then it equals None
data['att_feats'] = att_batch # if pre_ft is 0, then it equals None
data['images'] = img_batch # if pre_ft is 1, then it equals None
data[
'semantic_words'] = attrs_batch # if attributes is 1, then it equals None
data[
'semantic_words_prob'] = attrs_prob_batch # if attributes is 1, then it equals None
data['labels'] = label_batch
data['gts'] = gts
data['masks'] = mask_batch
data['bounds'] = {
'it_pos_now': self.iterators[split],
'it_max': len(split_ix),
'wrapped': wrapped
}
data['infos'] = infos
gc.collect()
return data
def run_trials(model_type, num_trials, dataset, selection_strategy, metric, C, alpha, \
bootstrap_size, balance, budget, step_size, topk, w_o, w_r, seed=0, lr_C=1, svm_C=1, svm_gamma=0, zaidan_C=0.01, zaidan_Ccontrast=1.0, zaidan_nu=1.0, Debug=False):
(X_pool, y_pool, X_test, y_test, feat_names) = load_dataset(dataset)
if not feat_names:
feat_names = np.arange(X_pool.shape[1])
feat_freq = np.diff(X_pool.tocsc().indptr)
fe = feature_expert(X_pool,
y_pool,
metric,
smoothing=1e-6,
C=C,
pick_only_top=True)
result = np.ndarray(num_trials, dtype=object)
for i in range(num_trials):
print '-' * 50
print 'Starting Trial %d of %d...' % (i + 1, num_trials)
trial_seed = seed + i # initialize the seed for the trial
training_set, pool_set = RandomBootstrap(X_pool, y_pool,
bootstrap_size, balance,
trial_seed)
# In order to get the best parameters
if 0:
# Train classifier
#
# For an initial search, a logarithmic grid with basis
# 10 is often helpful. Using a basis of 2, a finer
# tuning can be achieved but at a much higher cost.
C_range = 10.0**np.arange(-5, 9)
gamma_range = 10.0**np.arange(-5, 5)
param_grid = dict(gamma=gamma_range, C=C_range)
cv = StratifiedKFold(y=y_pool, n_folds=5)
grid = GridSearchCV(SVC(kernel='poly'),
param_grid=param_grid,
cv=cv)
grid.fit(X_pool, np.array(y_pool))
print("The best classifier is: ", grid.best_estimator_)
# Now we need to fit a classifier for all parameters in the 2d version
# (we use a smaller set of parameters here because it takes a while to train)
C_2d_range = [1, 1e2, 1e4]
gamma_2d_range = [1e-1, 1, 1e1]
classifiers = []
for C in C_2d_range:
for gamma in gamma_2d_range:
clf = SVC(C=C, gamma=gamma)
clf.fit(X_pool, np.array(y_pool))
classifiers.append((C, gamma, clf))
result[i] = learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, fe, \
selection_strategy, budget, step_size, topk, w_o, w_r, trial_seed, lr_C, svm_C, svm_gamma, zaidan_C, zaidan_Ccontrast, zaidan_nu, Debug)
return result, feat_names, feat_freq
if not os.path.exists(args.imgdir):
raise RuntimeError('Image directory not found')
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
fns = glob(args.imgdir + '/*.png')
fns.sort()
for fn in fns:
print 'working on image %s' % fn
image = cv2.imread(fn, cv2.IMREAD_GRAYSCALE)
gradient = np.gradient(image)
final_image = np.ndarray((image.shape[0], image.shape[0]), dtype='int')
if args.numcol != 256:
image = colorization(image, args.numcol)
delta_border = image.shape[1] - image.shape[0]
delta_border_per_pixel = float(delta_border) / image.shape[0]
for cnt, img in enumerate(image):
offset = int((cnt + 1) * delta_border_per_pixel)
offset = delta_border - offset
final_image[cnt, :] = img[offset:image.shape[0] + offset]
newfile = args.outdir + '/' + fn.split('/')[-1]
newfile = newfile.replace('.png', '_crop.png')
def __init__(self, graph_path, target_size=(320, 240), tf_config=None):
self.target_size = target_size
# load graph
logger.info('loading graph from %s(default size=%dx%d)' %
(graph_path, target_size[0], target_size[1]))
with tf.gfile.GFile(graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
self.graph = tf.get_default_graph()
tf.import_graph_def(graph_def, name='TfPoseEstimator')
self.persistent_sess = tf.Session(graph=self.graph, config=tf_config)
# for op in self.graph.get_operations():
# print(op.name)
# for ts in [n.name for n in tf.get_default_graph().as_graph_def().node]:
# print(ts)
self.tensor_image = self.graph.get_tensor_by_name(
'TfPoseEstimator/image:0')
self.tensor_output = self.graph.get_tensor_by_name(
'TfPoseEstimator/Openpose/concat_stage7:0')
self.tensor_heatMat = self.tensor_output[:, :, :, :19]
self.tensor_pafMat = self.tensor_output[:, :, :, 19:]
self.upsample_size = tf.placeholder(dtype=tf.int32,
shape=(2, ),
name='upsample_size')
self.tensor_heatMat_up = tf.image.resize_area(
self.tensor_output[:, :, :, :19],
self.upsample_size,
align_corners=False,
name='upsample_heatmat')
self.tensor_pafMat_up = tf.image.resize_area(
self.tensor_output[:, :, :, 19:],
self.upsample_size,
align_corners=False,
name='upsample_pafmat')
smoother = Smoother({'data': self.tensor_heatMat_up}, 25, 3.0)
gaussian_heatMat = smoother.get_output()
max_pooled_in_tensor = tf.nn.pool(gaussian_heatMat,
window_shape=(3, 3),
pooling_type='MAX',
padding='SAME')
self.tensor_peaks = tf.where(
tf.equal(gaussian_heatMat, max_pooled_in_tensor), gaussian_heatMat,
tf.zeros_like(gaussian_heatMat))
self.heatMat = self.pafMat = None
# warm-up
self.persistent_sess.run(
tf.variables_initializer([
v for v in tf.global_variables() if v.name.split(':')[0] in [
x.decode('utf-8') for x in self.persistent_sess.run(
tf.report_uninitialized_variables())
]
]))
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [
np.ndarray(shape=(target_size[1], target_size[0], 3),
dtype=np.float32)
],
self.upsample_size: [target_size[1], target_size[0]]
})
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [
np.ndarray(shape=(target_size[1], target_size[0], 3),
dtype=np.float32)
],
self.upsample_size: [target_size[1] // 2, target_size[0] // 2]
})
self.persistent_sess.run(
[self.tensor_peaks, self.tensor_heatMat_up, self.tensor_pafMat_up],
feed_dict={
self.tensor_image: [
np.ndarray(shape=(target_size[1], target_size[0], 3),
dtype=np.float32)
],
self.upsample_size: [target_size[1] // 4, target_size[0] // 4]
})
def overlap(cibra, ciket, nmo, nocc, s=None):
'''Overlap between two CISD wavefunctions.
Args:
s : 2D array
The overlap matrix of non-orthogonal one-particle basis
'''
if s is None:
return dot(cibra, ciket, nmo, nocc)
DEBUG = True
nvir = nmo - nocc
nov = nocc * nvir
bra0, bra1, bra2 = cisdvec_to_amplitudes(cibra, nmo, nocc)
ket0, ket1, ket2 = cisdvec_to_amplitudes(ciket, nmo, nocc)
# Sort the ket orbitals to make the orbitals in bra one-one mapt to orbitals
# in ket.
if ((not DEBUG) and abs(numpy.linalg.det(s[:nocc, :nocc]) - 1) < 1e-2
and abs(numpy.linalg.det(s[nocc:, nocc:]) - 1) < 1e-2):
ket_orb_idx = numpy.where(abs(s) > 0.9)[1]
s = s[:, ket_orb_idx]
oidx = ket_orb_idx[:nocc]
vidx = ket_orb_idx[nocc:] - nocc
ket1 = ket1[oidx[:, None], vidx]
ket2 = ket2[oidx[:, None, None, None], oidx[:, None, None],
vidx[:, None], vidx]
ooidx = numpy.tril_indices(nocc, -1)
vvidx = numpy.tril_indices(nvir, -1)
bra2aa = bra2 - bra2.transpose(1, 0, 2, 3)
bra2aa = lib.take_2d(bra2aa.reshape(nocc**2,
nvir**2), ooidx[0] * nocc + ooidx[1],
vvidx[0] * nvir + vvidx[1])
ket2aa = ket2 - ket2.transpose(1, 0, 2, 3)
ket2aa = lib.take_2d(ket2aa.reshape(nocc**2,
nvir**2), ooidx[0] * nocc + ooidx[1],
vvidx[0] * nvir + vvidx[1])
occlist0 = numpy.arange(nocc).reshape(1, nocc)
occlists = numpy.repeat(occlist0, 1 + nov + bra2aa.size, axis=0)
occlist0 = occlists[:1]
occlist1 = occlists[1:1 + nov]
occlist2 = occlists[1 + nov:]
ia = 0
for i in range(nocc):
for a in range(nocc, nmo):
occlist1[ia, i] = a
ia += 1
ia = 0
for i in range(nocc):
for j in range(i):
for a in range(nocc, nmo):
for b in range(nocc, a):
occlist2[ia, i] = a
occlist2[ia, j] = b
ia += 1
na = len(occlists)
if DEBUG:
trans = numpy.empty((na, na))
for i, idx in enumerate(occlists):
s_sub = s[idx].T.copy()
minors = s_sub[occlists]
trans[i, :] = numpy.linalg.det(minors)
# Mimic the transformation einsum('ab,ap->pb', FCI, trans).
# The wavefunction FCI has the [excitation_alpha,excitation_beta]
# representation. The zero blocks like FCI[S_alpha,D_beta],
# FCI[D_alpha,D_beta], are explicitly excluded.
bra_mat = numpy.zeros((na, na))
bra_mat[0, 0] = bra0
bra_mat[0, 1:1 + nov] = bra_mat[1:1 + nov, 0] = bra1.ravel()
bra_mat[0, 1 + nov:] = bra_mat[1 + nov:, 0] = bra2aa.ravel()
bra_mat[1:1 + nov, 1:1 + nov] = bra2.transpose(0, 2, 1,
3).reshape(nov, nov)
ket_mat = numpy.zeros((na, na))
ket_mat[0, 0] = ket0
ket_mat[0, 1:1 + nov] = ket_mat[1:1 + nov, 0] = ket1.ravel()
ket_mat[0, 1 + nov:] = ket_mat[1 + nov:, 0] = ket2aa.ravel()
ket_mat[1:1 + nov, 1:1 + nov] = ket2.transpose(0, 2, 1,
3).reshape(nov, nov)
ovlp = lib.einsum('ab,ap,bq,pq->', bra_mat, trans, trans, ket_mat)
else:
nov1 = 1 + nov
noovv = bra2aa.size
bra_SS = numpy.zeros((nov1, nov1))
bra_SS[0, 0] = bra0
bra_SS[0, 1:] = bra_SS[1:, 0] = bra1.ravel()
bra_SS[1:, 1:] = bra2.transpose(0, 2, 1, 3).reshape(nov, nov)
ket_SS = numpy.zeros((nov1, nov1))
ket_SS[0, 0] = ket0
ket_SS[0, 1:] = ket_SS[1:, 0] = ket1.ravel()
ket_SS[1:, 1:] = ket2.transpose(0, 2, 1, 3).reshape(nov, nov)
trans_SS = numpy.empty((nov1, nov1))
trans_SD = numpy.empty((nov1, noovv))
trans_DS = numpy.empty((noovv, nov1))
occlist01 = occlists[:nov1]
for i, idx in enumerate(occlist01):
s_sub = s[idx].T.copy()
minors = s_sub[occlist01]
trans_SS[i, :] = numpy.linalg.det(minors)
minors = s_sub[occlist2]
trans_SD[i, :] = numpy.linalg.det(minors)
s_sub = s[:, idx].copy()
minors = s_sub[occlist2]
trans_DS[:, i] = numpy.linalg.det(minors)
ovlp = lib.einsum('ab,ap,bq,pq->', bra_SS, trans_SS, trans_SS, ket_SS)
ovlp += lib.einsum('ab,a ,bq, q->', bra_SS, trans_SS[:, 0], trans_SD,
ket2aa.ravel())
ovlp += lib.einsum('ab,ap,b ,p ->', bra_SS, trans_SD, trans_SS[:, 0],
ket2aa.ravel())
ovlp += lib.einsum(' b, p,bq,pq->', bra2aa.ravel(), trans_SS[0, :],
trans_DS, ket_SS)
ovlp += lib.einsum(' b, p,b ,p ->', bra2aa.ravel(), trans_SD[0, :],
trans_DS[:, 0], ket2aa.ravel())
ovlp += lib.einsum('a ,ap, q,pq->', bra2aa.ravel(), trans_DS,
trans_SS[0, :], ket_SS)
ovlp += lib.einsum('a ,a , q, q->', bra2aa.ravel(), trans_DS[:, 0],
trans_SD[0, :], ket2aa.ravel())
# FIXME: whether to approximate the overlap between double excitation coefficients
if numpy.linalg.norm(bra2aa) * numpy.linalg.norm(ket2aa) < 1e-4:
# Skip the overlap if coefficients of double excitation are small enough
pass
if (abs(numpy.linalg.det(s[:nocc, :nocc]) - 1) < 1e-2
and abs(numpy.linalg.det(s[nocc:, nocc:]) - 1) < 1e-2):
# If the overlap matrix close to identity enough, use the <D|D'> overlap
# for orthogonal single-particle basis to approximate the overlap
# for non-orthogonal basis.
ovlp += numpy.dot(bra2aa.ravel(), ket2aa.ravel()) * trans_SS[0,
0] * 2
else:
from multiprocessing import sharedctypes, Process
buf_ctypes = sharedctypes.RawArray('d', noovv)
trans_ket = numpy.ndarray(noovv, buffer=buf_ctypes)
def trans_dot_ket(i0, i1):
for i in range(i0, i1):
s_sub = s[occlist2[i]].T.copy()
minors = s_sub[occlist2]
trans_ket[i] = numpy.linalg.det(minors).dot(ket2aa.ravel())
nproc = lib.num_threads()
if nproc > 1:
seg = (noovv + nproc - 1) // nproc
ps = []
for i0, i1 in lib.prange(0, noovv, seg):
p = Process(target=trans_dot_ket, args=(i0, i1))
ps.append(p)
p.start()
[p.join() for p in ps]
else:
trans_dot_ket(0, noovv)
ovlp += numpy.dot(bra2aa.ravel(), trans_ket) * trans_SS[0, 0] * 2
return ovlp
def noiselevel(self):
if len(self.img.shape) < 3:
self.img = np.expand_dims(self.img, 2)
nlevel = np.ndarray(self.img.shape[2])
th = np.ndarray(self.img.shape[2])
num = np.ndarray(self.img.shape[2])
kh = np.expand_dims(np.expand_dims(np.array([-0.5, 0, 0.5]), 0),2)
imgh = correlate(self.img, kh, mode='nearest')
imgh = imgh[:, 1: imgh.shape[1] - 1, :]
imgh = imgh * imgh
kv = np.expand_dims(np.vstack(np.array([-0.5, 0, 0.5])), 2)
imgv = correlate(self.img, kv, mode='nearest')
imgv = imgv[1: imgv.shape[0] - 1, :, :]
imgv = imgv * imgv
Dh = np.matrix(self.convmtx2(np.squeeze(kh,2), self.patchsize, self.patchsize))
Dv = np.matrix(self.convmtx2(np.squeeze(kv,2), self.patchsize, self.patchsize))
DD = Dh.getH() * Dh + Dv.getH() * Dv
r = np.double(np.linalg.matrix_rank(DD))
Dtr = np.trace(DD)
tau0 = gamma.ppf(self.conf, r / 2, scale=(2 * Dtr / r))
for cha in range(self.img.shape[2]):
X = view_as_windows(self.img[:, :, cha], (self.patchsize, self.patchsize))
X = X.reshape(np.int(X.size / self.patchsize ** 2), self.patchsize ** 2, order='F').transpose()
Xh = view_as_windows(imgh[:, :, cha], (self.patchsize, self.patchsize - 2))
Xh = Xh.reshape(np.int(Xh.size / ((self.patchsize - 2) * self.patchsize)),
((self.patchsize - 2) * self.patchsize), order='F').transpose()
Xv = view_as_windows(imgv[:, :, cha], (self.patchsize - 2, self.patchsize))
Xv = Xv.reshape(np.int(Xv.size / ((self.patchsize - 2) * self.patchsize)),
((self.patchsize - 2) * self.patchsize), order='F').transpose()
Xtr = np.expand_dims(np.sum(np.concatenate((Xh, Xv), axis=0), axis=0), 0)
if self.decim > 0:
XtrX = np.transpose(np.concatenate((Xtr, X), axis=0))
XtrX = np.transpose(XtrX[XtrX[:, 0].argsort(),])
p = np.floor(XtrX.shape[1] / (self.decim + 1))
p = np.expand_dims(np.arange(0, p) * (self.decim + 1), 0)
Xtr = XtrX[0, p.astype('int')]
X = np.squeeze(XtrX[1:XtrX.shape[1], p.astype('int')])
# noise level estimation
tau = np.inf
if X.shape[1] < X.shape[0]:
sig2 = 0
else:
cov = (np.asmatrix(X) @ np.asmatrix(X).getH()) / (X.shape[1] - 1)
d = np.flip(np.linalg.eig(cov)[0], axis=0)
sig2 = d[0]
for i in range(1, self.itr):
# weak texture selection
tau = sig2 * tau0
p = Xtr < tau
Xtr = Xtr[p]
X = X[:, np.squeeze(p)]
# noise level estimation
if X.shape[1] < X.shape[0]:
break
cov = (np.asmatrix(X) @ np.asmatrix(X).getH()) / (X.shape[1] - 1)
d = np.flip(np.linalg.eig(cov)[0], axis=0)
sig2 = d[0]
nlevel[cha] = np.sqrt(sig2)
th[cha] = tau
num[cha] = X.shape[1]
# clean up
self.img = np.squeeze(self.img)
return nlevel, th, num
def get_temp_3A(matrix, telemetry, satellite, Cs):
satelite = satellite
c1 = 1.1910427e-5
c2 = 1.4387752
print("largo telemetria: ", len(telemetry))
Ns = satelite.Ns[2]
b0 = satelite.b0[2]
b1 = satelite.b1[2]
b2 = satelite.b2[2]
print("Ns", Ns)
print("bo", b0)
print("b1", b1)
d = np.matrix(satelite.d)
d0 = d[:, 0]
d1 = d[:, 1]
d2 = d[:, 2]
CPRO = 4 * telemetry[9]
CPR1 = 4 * telemetry[10]
CPR2 = 4 * telemetry[11]
CPR3 = 4 * telemetry[12]
print(CPRO)
T0 = d0[0] + d1[0] * CPRO + d2[0] * CPRO**2
T1 = d0[1] + d1[1] * CPR1 + d2[1] * CPR1**2
T2 = d0[2] + d1[2] * CPR2 + d2[2] * CPR2**2
T3 = d0[3] + d1[3] * CPR3 + d2[3] * CPR3**2
print(T0)
Tbb = .25 * (T0 + T1 + T2 + T3)
Tbb = satelite.A[2] + Tbb * satelite.B[2]
Tbb = Tbb[0, 0]
vc = satelite.vc[2]
print("vc", vc)
print("Tbb", Tbb)
Nbb = c1 * vc**3 / (math.exp(c2 * vc / Tbb) - 1.0)
Cb = telemetry[14] * 4
print("Cb", Cb)
print("Cs", Cs)
matrix_therm = np.ndarray((matrix.shape[0], matrix.shape[1]))
for i in range(0, matrix.shape[0]):
for j in range(0, matrix.shape[1]):
Ce = np.array(matrix[i, j]).astype(np.float64)
N1 = Ns + (Nbb - Ns) * (Cs - Ce * 4) / (Cs - Cb)
Nc = b0 + b1 * N1 + b2 * N1**2
Ne = N1 + Nc
T = c2 * vc / math.log(abs(c1 * (vc**3) / Ne + 1.0))
T = (T - satelite.A[2]) / (satelite.B[2])
T = (T - 273.15 +
150) / 200.0 * 256.0 # range 0-255 for -150 +50 celcius
matrix_therm[i, j] = T
return matrix_therm
out_fields = '_fields_list.csv'
out_residuals = '_template_residuals_median.csv'
out_residuals_abs = '_template_residuals_median_abs.csv'
if not os.path.isfile(out_residuals):
new_txt_file(out_fields)
new_txt_file(out_residuals)
new_txt_file(out_residuals_abs)
for field_id in selected_observation_fields: # filter by date
print 'Working on field ' + str(field_id)
spectra_row = np.where(field_id == observation_fields)
# initialize plot
n_in_field = len(spectra_row[0])
if n_in_field < 100:
continue
# create a stack of field residuals
field_residuals = np.ndarray((n_in_field, len(wvl_read_finer)))
# init plot
fig, axes = plt.subplots(2, 1)
for i_row in range(len(spectra_row[0])):
row = spectra_row[0][i_row]
object_param = galah_param[row]
object_spectra = spectral_data[row]
# get template spectra
template_file = get_best_match(object_param['teff_guess'],
object_param['logg_guess'],
object_param['feh_guess'],
grid_list,
midpoint=False) + '.csv'
template_spectra = np.loadtxt(galah_template_dir + template_file,
delimiter=',')[idx_read]
# subtract spectra
def comparison_v2(adata,
name_keys,
group=None,
color_thresholds=None,
n_genes=70):
name_list_cut = {}
for i, j in enumerate(name_keys):
name_list_cut[i] = adata.uns[j][0:n_genes]
name_list = {}
for i, j in enumerate(name_keys):
name_list[i] = adata.uns[j]
length = n_genes
width = len(name_list)
rank_table = pd.DataFrame(name_list_cut)
row_names = np.arange(n_genes) + 1
colors = np.ndarray((length, width))
for key in name_list:
for i in range(n_genes):
top100 = False
top50 = False
top20 = False
for key2 in name_list:
if key is key2:
pass
else:
if name_list[key][i] in name_list[key2]:
index = name_list[key2].index(name_list[key][i])
if index < 100:
top100 = True
if index < 50:
top50 = True
if index >= 20:
top20 = True
else:
pass
else:
top100 = False
top50 = False
top20 = False
if top100 is True:
colors[i, key] = 0.55
if top50 is True:
colors[i, key] = 0.75
if top20 is True:
colors[i, key] = 0.9
else:
colors[i, :] = 0.35
plt.figure(figsize=(4, 4), dpi=120)
ax = plt.subplot(111, frame_on=False)
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
ax.table(cellText=rank_table.as_matrix(),
rowLabels=row_names,
colLabels=name_keys,
cellColours=cm.afmhot(colors),
loc="center",
fontsize=22)
plt.show()
def get_lsb(img: np.ndarray) -> np.ndarray:
test = np.ndarray(img.shape, dtype=np.bool)
for i in range(img.shape[0]):
for j in range(img.shape[1]):
test[i, j] = round(img[i, j]) % 2
return test
allow_soft_placement=True)) as sess:
restore_path = os.path.join(old_checkpoint_path,
'model_epoch%d.ckpt' % (restore_epoch))
print "restoring from ckpt: {}...".format(restore_path)
saver.restore(sess, restore_path)
print "Start Evaluation"
if EVAL_TRAIN:
print("{} on training set...".format(datetime.now()))
ls = 0.
mIoU = 0.
cIoU = np.zeros((num_classes, ), dtype=np.float32)
count = 0
out = np.ndarray(
[train_generator.data_size, height, width, num_classes],
dtype=np.float32)
for step in range(train_betches_per_epoch):
count += 1
print('step number: {}'.format(step))
# Get a batch of images and labels
batch_xs, batch_ys = train_generator.next_batch(batch_size)
# And run the data
result = sess.run([softmax_maps, meanIoU, loss] + classIoU,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_prob: 1.
})
out[step * batch_size:(step + 1) * batch_size] = result[0]
mIoU += result[1]
def __init__(self, policy_controller=None):
""""""
super().__init__(toggle_key='m', policy_controller=policy_controller)
self.is_on = True
self.frames = []
self.detection_mask = np.ndarray((0, ))
def add_mesh(self, v, f, c=None, uv=None, shading={}):
sh = self.__get_shading(shading)
mesh_obj = {}
#it is a tet
if v.shape[1] == 3 and f.shape[1] == 4:
f_tmp = np.ndarray([f.shape[0]*4, 3], dtype=f.dtype)
for i in range(f.shape[0]):
f_tmp[i*4+0] = np.array([f[i][1], f[i][0], f[i][2]])
f_tmp[i*4+1] = np.array([f[i][0], f[i][1], f[i][3]])
f_tmp[i*4+2] = np.array([f[i][1], f[i][2], f[i][3]])
f_tmp[i*4+3] = np.array([f[i][2], f[i][0], f[i][3]])
f = f_tmp
if v.shape[1] == 2:
v = np.append(v, np.zeros([v.shape[0], 1]), 1)
# Type adjustment vertices
v = v.astype("float32", copy=False)
# Color setup
colors, coloring = self.__get_colors(v, f, c, sh)
# Type adjustment faces and colors
c = colors.astype("float32", copy=False)
# Material and geometry setup
ba_dict = {"color": p3s.BufferAttribute(c)}
if coloring == "FaceColors":
verts = np.zeros((f.shape[0]*3, 3), dtype="float32")
for ii in range(f.shape[0]):
#print(ii*3, f[ii])
verts[ii*3] = v[f[ii,0]]
verts[ii*3+1] = v[f[ii,1]]
verts[ii*3+2] = v[f[ii,2]]
v = verts
else:
f = f.astype("uint32", copy=False).ravel()
ba_dict["index"] = p3s.BufferAttribute(f, normalized=False)
ba_dict["position"] = p3s.BufferAttribute(v, normalized=False)
if type(uv) != type(None):
uv = (uv - np.min(uv)) / (np.max(uv) - np.min(uv))
tex = p3s.DataTexture(data=gen_checkers(20, 20), format="RGBFormat", type="FloatType")
material = p3s.MeshStandardMaterial(map=tex, reflectivity=sh["reflectivity"], side=sh["side"],
roughness=sh["roughness"], metalness=sh["metalness"], flatShading=sh["flat"],
polygonOffset=True, polygonOffsetFactor= 1, polygonOffsetUnits=5)
ba_dict["uv"] = p3s.BufferAttribute(uv.astype("float32", copy=False))
else:
material = p3s.MeshStandardMaterial(vertexColors=coloring, reflectivity=sh["reflectivity"],
side=sh["side"], roughness=sh["roughness"], metalness=sh["metalness"],
flatShading=sh["flat"],
polygonOffset=True, polygonOffsetFactor= 1, polygonOffsetUnits=5)
geometry = p3s.BufferGeometry(attributes=ba_dict)
if coloring == "VertexColors":
geometry.exec_three_obj_method('computeVertexNormals')
else:
geometry.exec_three_obj_method('computeFaceNormals')
# Mesh setup
mesh = p3s.Mesh(geometry=geometry, material=material)
# Wireframe setup
mesh_obj["wireframe"] = None
if sh["wireframe"]:
wf_geometry = p3s.WireframeGeometry(mesh.geometry) # WireframeGeometry
wf_material = p3s.LineBasicMaterial(color=sh["wire_color"], linewidth=sh["wire_width"])
wireframe = p3s.LineSegments(wf_geometry, wf_material)
mesh.add(wireframe)
mesh_obj["wireframe"] = wireframe
# Bounding box setup
if sh["bbox"]:
v_box, f_box = self.__get_bbox(v)
_, bbox = self.add_edges(v_box, f_box, sh, mesh)
mesh_obj["bbox"] = [bbox, v_box, f_box]
# Object setup
mesh_obj["max"] = np.max(v, axis=0)
mesh_obj["min"] = np.min(v, axis=0)
mesh_obj["geometry"] = geometry
mesh_obj["mesh"] = mesh
mesh_obj["material"] = material
mesh_obj["type"] = "Mesh"
mesh_obj["shading"] = sh
mesh_obj["coloring"] = coloring
mesh_obj["arrays"] = [v, f, c] # TODO replays with proper storage or remove if not needed
return self.__add_object(mesh_obj)
def fit(self, data, labels=None):
"""
Parameters
----------
data : dataframe - The data to use for the estimation (in sorted by ID in compact format)
labels: an optional dictionary for relabeling column names
Returns
-------
matrix_set : An estimated transition matrix set
Notes
------
* loop over data rows (id, timepoint, state)
* at least two distinct timepoints are required (initial and final)
* calculate population count N^i_k per state i per timepoint k
* calculate migrations count N^{ij}_{kl} from i to j from timepoint k to timepoint l
* calculate transition matrix as ratio T^{ij}_{kl} = N^{ij}_{kl} / N^i_k
* calculate also count-averaged matrix
References
----------
"""
# Allow for flexible labelling for dataframe columns
if labels is not None:
state_label = labels['State']
timestep_label = labels['Time']
id_label = labels['ID']
else:
state_label = 'State'
id_label = 'ID'
timestep_label = 'Time'
# Old way of enumerating cohort intervals was using labels
# cohort_labels = data[timestep_label].unique()
# cohort_dim = len(cohort_labels) - 1
# The size of the state space
state_dim = self.states.cardinality
# The number of cohorts is the number of intervals
# Minimally two (initial and final)
cohort_dim = len(self.cohort_bounds) - 1
event_count = data[id_label].count()
# store data in 1d arrays for faster processing
# capture nan events for missing observations
event_exists = np.empty(event_count, int)
entity_id = np.empty(event_count, int)
entity_state = np.empty(event_count, int)
event_time = np.empty(event_count, int)
nan_count = 0
i = 0
for index, row in data.iterrows():
entity_id[i] = row[id_label]
try:
entity_state[i] = row[state_label]
event_time[i] = row[timestep_label]
event_exists[i] = 1 # indicates a valid (complete) data row
except ValueError:
entity_state[i] = -99999
event_time[i] = -99999
event_exists[i] = 0
nan_count += 1
i += 1
self.nans = nan_count
# store number of entities observed in given state per time step
tm_count = np.ndarray((state_dim, cohort_dim + 1), int)
# store number of entities observed to transition from state (From) to state (To) per period
tmn_count = np.ndarray((state_dim, state_dim, cohort_dim), int)
# store normalized frequencies
tmn_values = np.ndarray((state_dim, state_dim, cohort_dim), float)
# matrix to store average transitions
tmn_average = np.ndarray((state_dim, state_dim), float)
# initialize to zero (TODO ?)
tm_count.fill(0)
tmn_count.fill(0)
tmn_values.fill(0)
tmn_average.fill(0)
# TODO Capture case if entity with only one observation (hence no transition count)
# TODO Capture case with stale observations (no transitions)
for i in range(0, event_count - 1): # the last point handled separately
if event_exists[i] == 1:
# while processing valid event data from same entity
# increment state count
tm_count[(entity_state[i], event_time[i])] += 1
if entity_id[i + 1] == entity_id[i]:
# increment migration count if there is subsequent observation
# NB: It does not have to be different
tmn_count[(entity_state[i], entity_state[i + 1], event_time[i])] += 1
# handle boundary cases
# the last event must be evaluated in comparison with its previous one
i = event_count - 1
if event_exists[i] == 1:
# ATTN we must shift the time index of the tm_count, tmn_count
tm_count[(entity_state[i], event_time[i] - 1)] += 1
if entity_id[i] == entity_id[i - 1]:
tmn_count[(entity_state[i - 1], entity_state[i], event_time[i] - 1)] += 1
# print(tm_count)
# print(tm_count.sum())
# print(tmn_count[:, :, 0])
self.counts = int(tm_count.sum())
# Normalization of counts to produce a family of probability matrices
for s1 in range(state_dim):
for s2 in range(state_dim):
for k in range(cohort_dim):
if tm_count[(s1, k)] > 0:
tmn_values[(s1, s2, k)] = tmn_count[(s1, s2, k)] / tm_count[(s1, k)]
# for k in range(cohort_dim):
# m = transitionMatrix.TransitionMatrix(tmn_values[:, :, k])
# m.print_matrix(accuracy=3)
# Average transition matrix (assuming temporal homogeneity)
for s1 in range(state_dim):
for s2 in range(state_dim):
tm_total_count = 0
for k in range(cohort_dim):
tmn_average[(s1, s2)] += tmn_count[(s1, s2, k)]
tm_total_count += tm_count[(s1, k)]
if tm_total_count > 0:
tmn_average[(s1, s2)] /= tm_total_count
self.average_matrix = tmn_average
# Confidence Interval Estimation (Based on Counts)
confint_lower = np.ndarray((state_dim, state_dim, cohort_dim))
confint_upper = np.ndarray((state_dim, state_dim, cohort_dim))
for k in range(cohort_dim - 1):
for s1 in range(state_dim):
intervals = st.multinomial_proportions_confint(tmn_count[s1, :, k], alpha=self.ci_alpha,
method=self.ci_method)
for s2 in range(state_dim):
confint_lower[s1, s2, k] = intervals[s2][0]
confint_upper[s1, s2, k] = intervals[s2][1]
self.confint_lower = confint_lower
self.confint_upper = confint_upper
# Return a list of transition matrices
# Both absolute (frequency) and relative (probability) format
for k in range(cohort_dim):
self.matrix_set.append(tmn_values[:, :, k])
self.count_set.append(tmn_count[:, :, k])
# Return absolute counts at time points
for k in range(cohort_dim + 1):
self.count_normalization.append(tm_count[:, k])
# print(self.count_normalization)
return self.matrix_set
def run_multinomial_logistic_regression(train_subset=45000,
valid_size=5000,
test=True):
"""
In Multinomial Logistic Regression, we have
input X of (n X image_size * image_size * color_channel) dimension and
output Y of (n X num_labels) dimension, and Y is defined as:
Y = softmax( X * W + b )
where W and b are weights and biases. The loss function is defined as:
Loss = cross_entropy(Y, labels)
We use stochastic gradient descent, with batch size of 128, learning rate of 0.5 and 3001 steps.
We do not use any regularization because it does not improve the accuracy for this case.
At the end of the training, accuracy curve, loss curve will be plotted.
Keyword arguments:
train_subset -- the number of training example
valid_size -- number data in validation set
test -- if true, output a .csv file that predict 300000 data in testing set
"""
train_dataset, train_labels, valid_dataset, valid_labels = \
get_train_valid_data(train_subset, valid_size)
print 'Building graph...'
batch_size = 128
graph = tf.Graph()
with graph.as_default():
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, num_features))
tf_train_labels = tf.placeholder(tf.float32,
shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_valid_labels = tf.constant(valid_labels)
weights = tf.Variable(tf.truncated_normal([num_features, num_labels]))
biases = tf.Variable(tf.zeros([num_labels]))
train_logits = model(tf_train_dataset, weights, biases)
train_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(train_logits,
tf_train_labels))
train_prediction = tf.nn.softmax(train_logits)
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(train_loss)
# Predictions for the training, validation, and test data.
valid_logits = model(tf_valid_dataset, weights, biases)
valid_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(valid_logits,
tf_valid_labels))
valid_prediction = tf.nn.softmax(valid_logits)
print 'Training...'
num_steps = 3001
trained_weights = np.ndarray(shape=(num_features, num_labels))
trained_biases = np.ndarray(shape=(num_labels))
train_losses = []
valid_losses = []
train_accuracies = []
valid_accuracies = []
with tf.Session(graph=graph) as session:
tf.initialize_all_variables().run()
print 'Initialized'
for step in xrange(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {
tf_train_dataset: batch_data,
tf_train_labels: batch_labels
}
_, tl, vl, predictions, trained_weights, trained_biases = session.run(
[
optimizer, train_loss, valid_loss, train_prediction,
weights, biases
],
feed_dict=feed_dict)
train_losses.append(tl)
valid_losses.append(vl)
train_accuracies.append(accuracy(predictions, batch_labels))
valid_accuracies.append(
accuracy(valid_prediction.eval(), valid_labels))
if step % 100 == 0:
print('Complete %.2f %%' % (float(step) / num_steps * 100.0))
# Plot losses and accuracies
print_loss(train_losses[-1], valid_losses[-1])
print_accuracy(train_accuracies[-1], valid_accuracies[-1])
plot(train_losses, valid_losses, 'Iteration', 'Loss')
plot(train_accuracies, valid_accuracies, 'Iteration', 'Accuracy')
if not test:
return train_losses[-1], valid_losses[-1]
part_size = 50000
test_graph = tf.Graph()
with test_graph.as_default():
tf_test_dataset = tf.placeholder(tf.float32,
shape=(part_size, num_features))
weights = tf.constant(trained_weights)
biases = tf.constant(trained_biases)
logits = model(tf_test_dataset, weights, biases)
test_prediction = tf.nn.softmax(logits)
test_dataset = load_test_data()
test_dataset = reformat_dataset(test_dataset)
total_part = 6
test_predicted_labels = np.ndarray(shape=(300000, 10))
for i in range(total_part):
test_dataset_part = test_dataset[i * part_size:(i + 1) * part_size]
with tf.Session(graph=test_graph) as session:
tf.initialize_all_variables().run()
feed_dict = {tf_test_dataset: test_dataset_part}
predict = session.run([test_prediction], feed_dict=feed_dict)
test_predicted_labels[i * part_size:(i + 1) *
part_size, :] = np.asarray(predict)[0]
test_predicted_labels = np.argmax(test_predicted_labels, 1)
label_matrices_to_csv(test_predicted_labels, 'submission.csv')
def __init__(self, x, y):
super().__init__()
self.coords = np.ndarray((0, 2))
self.x = x
self.y = y
def _execute(self, payload):
frame = payload.original_frame
raw_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
frames = self.frames
median = np.median(
frames, axis=0).astype(float) if len(frames) else np.ndarray(
(0, )) # .reshape(100, 100)
if len(self.detection_mask) == 0:
self.detection_mask = np.zeros(raw_gray.shape, dtype='uint8')
# df = payload.dfs.get('detections', pd.DataFrame()).drop_duplicates()
# if len(df) > 0:
vehicle_detections = list(payload.vehicle_detections)
boxes = (d.bounding_box for d in vehicle_detections)
# boxes = (b.get_scaled(0.5) for b in boxes)
bb_h_percentage = 0.2
boxes = [
BoundingBox(b.x, int(round(b.y + b.h * (1 - bb_h_percentage))),
b.w, int(round(b.h * bb_h_percentage))) for b in boxes
]
for box in boxes:
# y_start = max(0, box.y)
# y_end = max(box.y + box.h, 0)
# x_start = max(0, box.x)
# x_end = max(0, box.x + box.w)
# self.detection_mask[y_start: y_end, x_start:x_end] = 1
self.detection_mask[box.y:box.y + box.h, box.x:box.x + box.w] = 1
# cv2.imshow('detection_mask',self.detection_mask*255)
# cv2.waitKey(0)
gray = raw_gray.copy() # * (1 - self.detection_mask)
if len(median):
idxs = np.where(self.detection_mask == 1)
detections_median = np.median(median[idxs])
gray[idxs] = detections_median
self.frames.append(gray)
title = 'Median'
cv2.namedWindow(title, cv2.WINDOW_NORMAL)
if len(median):
cv2.imshow(title, median / 255)
# cv2.imshow(title, frame)
# cv2.imshow(title, gray / 255)
# cv2.imwrite(r'median.jpg', median, )
return payload
src = median / 255
from experimental import demo_erosion_dilatation
src = (1 - src) * 255
demo_erosion_dilatation(src, iterations=2)
erosion_size = 5
erosion_type = cv2.MORPH_ELLIPSE
element = cv2.getStructuringElement(
erosion_type, (2 * erosion_size + 1, 2 * erosion_size + 1),
(erosion_size, erosion_size))
erosion_dst = cv2.erode(src, element, iterations=1)
cv2.imshow('erosion', erosion_dst)
erosion_dst = cv2.dilate(src, element, iterations=2)
cv2.imshow('dialation', erosion_dst)
cv2.waitKey(0)
element = cv2.getStructuringElement(
erosion_type, (2 * erosion_size + 1, 2 * erosion_size + 1),
(erosion_size, erosion_size))
cv2.waitKey(1)
# ============================================
return payload
foiir_coefficient = 0.020 ##From Jon
pixelWidth = 100
pixelHeight = 100
pixelMaxSize = 1000
pixelmaxVoltage = 30
voltage2pixel = 255 / 30
number_snapshots = 300
x_dim = int(input("Size of X-dimension(pixels): "))
y_dim = int(input("Size of Y-dimension(pixels): "))
freq = int(input("Frequency of capture (Hz): "))
time = int(input("Length of time record (seconds): "))
seed = int(input("Set Seed: "))
number_snapshots = freq * time
data_cube = np.ndarray([y_dim, x_dim, number_snapshots])
array_past2 = np.ndarray([n_samples, y_dim, x_dim])
random.seed(seed)
text_file = open("OPIR_test/Data_header.txt", "w")
text_file.write("%d \n" % x_dim)
text_file.write("%d \n" % y_dim)
text_file.write("%d \n" % freq)
text_file.write("%d \n" % time)
text_file.write("%d \n" % seed)
noise = int(input("Noise = 1, No-Noise = 2: "))
coarse = 0
filter = 0
if noise == 1:
coarse = int(input("Coarse Noise= 1, Fine = 2: "))
#This example is directly copied from the Tensorflow examples provided from the Teachable Machine.
import tensorflow.keras
from PIL import Image, ImageOps
import numpy as np
# Disable scientific notation for clarity
np.set_printoptions(suppress=True)
# Load the model
model = tensorflow.keras.models.load_model('keras_model.h5')
# Create the array of the right shape to feed into the keras model
# The 'length' or number of images you can put into the array is
# determined by the first position in the shape tuple, in this case 1.
data = np.ndarray(shape=(1, 224, 224, 3), dtype=np.float32)
# Replace this with the path to your image
image = Image.open('/home/pi/openCV-examples/data/test.jpg')
#resize the image to a 224x224 with the same strategy as in TM2:
#resizing the image to be at least 224x224 and then cropping from the center
size = (224, 224)
image = ImageOps.fit(image, size, Image.ANTIALIAS)
#turn the image into a numpy array
image_array = np.asarray(image)
# display the resized image
image.show()
def main(block_name):
for pickle_file in glob.glob(sys.argv[1] + block_name + "/*.pickle"):
subject = pickle_file[len(pickle_file) - 12:len(pickle_file) - 7]
batch_size = 25
patch_size = 5 # filter size
myInitializer = None
if (block_name == "face"):
image_size_h = 72
image_size_w = 52
elif (block_name == "mouth"):
image_size_h = 24
image_size_w = 40
elif (block_name == "eye"):
image_size_h = 24
image_size_w = 32
elif (block_name == "topnose"):
image_size_h = 36
image_size_w = 40
elif (block_name == "nosetip"):
image_size_h = 32
image_size_w = 40
num_labels = 7 #the output of the network (7 neuron)
#num_channels = 3 # colour images have 3 channels
num_channels = 1 # grayscale images have 1 channel
# Load the pickle file containing the dataset
with open(pickle_file, 'rb') as f:
save = pickle.load(f)
train_dataset = save['training_dataset']
train_labels = save['training_emotion_label']
valid_dataset = save['validation_dataset']
valid_labels = save['validation_emotion_label']
test_dataset = save['test_dataset']
test_labels = save['test_emotion_label']
del save # hint to help gc free up memory
# Here I print the dimension of the three datasets
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
train_dataset = train_dataset.reshape(
(-1, image_size_w, image_size_h, num_channels)).astype(np.float32)
train_labels = train_labels.reshape((-1)).astype(np.ndarray)
valid_dataset = valid_dataset.reshape(
(-1, image_size_w, image_size_h, num_channels)).astype(np.float32)
valid_labels = valid_labels.reshape((-1)).astype(np.ndarray)
test_dataset = test_dataset.reshape(
(-1, image_size_w, image_size_h, num_channels)).astype(np.float32)
test_labels = test_labels.reshape((-1)).astype(np.ndarray)
# create the arrays from string, removing brackets as well
train_labels_new = extractArraysRemoveBrackets(train_labels)
valid_labels_new = extractArraysRemoveBrackets(valid_labels)
test_labels_new = extractArraysRemoveBrackets(test_labels)
train_dataset = image_histogram_equalization(train_dataset)
valid_dataset = image_histogram_equalization(valid_dataset)
test_dataset = image_histogram_equalization(test_dataset)
train_dataset = minmax_normalization(train_dataset)
valid_dataset = minmax_normalization(valid_dataset)
test_dataset = minmax_normalization(test_dataset)
#Printing the new shape of the datasets
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels_new.shape)
print('Test set', test_dataset.shape, test_labels_new.shape)
#Declaring the graph object necessary to build the model
graph = tf.Graph()
with graph.as_default():
print("Init Tensorflow variables...")
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, image_size_w,
image_size_h,
num_channels))
tf_train_labels = tf.placeholder(tf.float32,
shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Conv layer
# [patch_size, patch_size, num_channels, depth]
#conv1_weights = tf.Variable(tf.truncated_normal([patch_size, patch_size, num_channels, 6], stddev=0.1), name="conv1y_w")
conv1_weights = tf.get_variable(
name="conv1y_w",
shape=[patch_size, patch_size, num_channels, 6],
initializer=myInitializer)
conv1_biases = tf.Variable(tf.zeros([6]), name="conv1y_b")
# Conv layer
# [patch_size, patch_size, depth, depth]
conv2_weights = tf.get_variable(
name="conv2y_w",
shape=[patch_size, patch_size, 6, 12],
initializer=myInitializer)
conv2_biases = tf.Variable(tf.zeros([12]), name="conv2y_b")
# Output layer
conv1_size_w = (image_size_w - patch_size + 1) / 2
conv2_size_w = (conv1_size_w - patch_size + 1) / 2
conv1_size_h = (image_size_h - patch_size + 1) / 2
conv2_size_h = (conv1_size_h - patch_size + 1) / 2
dense1_weights = tf.get_variable(
name="dense1y_w",
shape=[conv2_size_w * conv2_size_h * 12, 256],
initializer=myInitializer)
dense1_biases = tf.Variable(tf.zeros([256], name="dense1y_b"))
# Output layer
layer_out_weights = tf.get_variable(name="outy_w",
shape=[256, num_labels],
initializer=myInitializer)
layer_out_biases = tf.Variable(tf.zeros(shape=[num_labels]),
name="outy_b")
# dropout (keep probability) - not used really up to now
keep_prob = tf.placeholder(tf.float32)
model_output = model(tf_train_dataset, image_size_w, image_size_h,
num_channels, conv1_weights, conv1_biases,
conv2_weights, conv2_biases, dense1_weights,
dense1_biases, layer_out_weights,
layer_out_biases, keep_prob)
loss = tf.reduce_mean(
tf.reduce_sum(tf.square(model_output - tf_train_labels)))
loss_summ = tf.summary.scalar("loss", loss)
global_step = tf.Variable(
0, trainable=False) # count the number of steps taken.
learning_rate = tf.train.exponential_decay(0.00125,
global_step,
300,
0.5,
staircase=True)
lrate_summ = tf.summary.scalar(
"learning rate",
learning_rate) #save in a summary for Tensorboard
optimizer = tf.train.GradientDescentOptimizer(
learning_rate).minimize(loss, global_step=global_step)
train_prediction = model_output
valid_prediction = model(tf_valid_dataset, image_size_w,
image_size_h, num_channels, conv1_weights,
conv1_biases, conv2_weights, conv2_biases,
dense1_weights, dense1_biases,
layer_out_weights, layer_out_biases)
test_prediction = model(tf_test_dataset, image_size_w,
image_size_h, num_channels, conv1_weights,
conv1_biases, conv2_weights, conv2_biases,
dense1_weights, dense1_biases,
layer_out_weights, layer_out_biases)
saver = tf.train.Saver()
total_epochs = 500
with tf.Session(graph=graph) as session:
merged_summaries = tf.summary.merge_all()
now = datetime.datetime.now()
log_path = "./sessions/summary_log/summaries_logs_p" + subject + str(
now.hour) + str(now.minute) + str(now.second)
writer_summaries = tf.summary.FileWriter(
log_path, session.graph)
tf.global_variables_initializer().run()
epochs = np.ndarray(0, int)
losses = np.ndarray(0, np.float32)
accuracy_batch = np.ndarray(0, np.float32)
accuracy_valid = np.ndarray(0, np.float32)
start = timer()
for epoch in range(total_epochs):
batch = create_batch(train_dataset, train_labels_new)
batch_data = batch[0]
batch_labels = batch[1]
feed_dict = {
tf_train_dataset: batch_data,
tf_train_labels: batch_labels,
keep_prob: 1.0
}
_, l, predictions, my_summary = session.run(
[optimizer, loss, model_output, merged_summaries],
feed_dict=feed_dict)
writer_summaries.add_summary(my_summary, epoch)
epochs = np.append(epochs, int(epoch + 1))
losses = np.append(losses, l)
accuracy_batch = np.append(
accuracy_batch,
accuracy(predictions, batch_labels, False))
accuracy_valid = np.append(
accuracy_valid,
accuracy(valid_prediction.eval(), valid_labels_new,
False))
'''
if (epoch % 50 == 0):
print("")
print("Loss at epoch: ", epoch, " is " , l)
print("Global Step: " + str(global_step.eval()) + " of " + str(total_epochs))
print("Learning Rate: " + str(learning_rate.eval()))
print("Minibatch size: " + str(batch_labels.shape))
print("Validation size: " + str(valid_labels_new.shape))
accuracy(predictions, batch_labels, True)
print("")
'''
end = timer()
sessionTime = end - start
saver.save(session,
"./sessions/tensorflow/cnn_arch1_pitch_p" + subject,
global_step=epoch) # save the session
accuracy_test = accuracy(test_prediction.eval(),
test_labels_new, True)
output = np.column_stack(
(epochs.flatten(), losses.flatten(),
accuracy_batch.flatten(), accuracy_valid.flatten()))
np.savetxt(
"./sessions/epochs_log/subject_" + subject + ".txt",
output,
header="epoch loss accuracy_batch accuracy_valid",
footer="accuracy_test:\n" + str(accuracy_test) +
"\ntime:\n" + str(sessionTime),
delimiter=' ')
print("# Test size: " + str(test_labels_new.shape))
regressor = GaussianProcessRegressor(copy_X_train=False, alpha=0.01778279410038923,
kernel=kernels.RationalQuadratic(alpha=1, length_scale=1), n_restarts_optimizer=4, normalize_y=False)
"""
regressor = GaussianProcessRegressor(copy_X_train=False)
parameters = {'kernel':(kernels.RationalQuadratic(), kernels.RBF(), kernels.WhiteKernel()),
'alpha': np.logspace(-10, 1, 5),
'n_restarts_optimizer': range(1,5),
'normalize_y': [True, False]}
clf = GridSearchCV(regressor, parameters, scoring=rmsle_scorer, verbose=10)
X_train, y_train = resample(X, y, n_samples=500)
clf.fit(X_train, y_train)
print("best_estimator_:", clf.best_estimator_)
print("best_score_:", clf.best_score_)
print("best_params_:", clf.best_params_)
print("best_score_:", clf.best_score_)
regressor.set_params(**clf.best_params_)
"""
X_train, y_train = resample(X, y, n_samples=5000)
regressor.fit(X_train, y_train)
print("Training done, testing...")
# Since we can't load the whole dataset, do batch testing
batch_size = 5000
X_test, y_test = resample(X, y, n_samples=100000)
y_pred = np.ndarray((0,))
for i in range(0, X_test.shape[0], batch_size):
y_pred = np.hstack((y_pred, regressor.predict(X_test[i: i + batch_size])))
print("RMSLE =", root_mean_squared_log_error(y_test, y_pred)) # Last result: 0.469685
def pool_bc01(imgs,
poolout,
switches,
pool_h, pool_w, stride_y, stride_x):
""" Multi-image, multi-channel pooling
imgs has shape (n_imgs, n_channels, img_h, img_w)
poolout has shape (n_imgs, n_channels, img_h//stride_y, img_w//stride_x)
switches has shape (n_imgs, n_channels, img_h//stride_y, img_w//stride_x, 2)
"""
# TODO: mean pool
print "pool"
imgs = ndarray(shape=imgs.shape, dtype=float, buffer=imgs)
poolout = ndarray(shape=poolout.shape, dtype=float, buffer=poolout)
switches = ndarray(shape=switches.shape, dtype=int, buffer=switches)
n_imgs = imgs.shape[0]
n_channels = imgs.shape[1]
img_h = imgs.shape[2]
img_w = imgs.shape[3]
out_h = img_h // stride_y
out_w = img_w // stride_x
pool_h_top = pool_h // 2 - 1 + pool_h % 2
pool_h_bottom = pool_h // 2 + 1
pool_w_left = pool_w // 2 - 1 + pool_w % 2
pool_w_right = pool_w // 2 + 1
if not n_imgs == poolout.shape[0] == switches.shape[0]:
raise ValueError('Mismatch in number of images.')
if not n_channels == poolout.shape[1] == switches.shape[1]:
raise ValueError('Mismatch in number of channels.')
if not (out_h == poolout.shape[2] == switches.shape[2] and out_w == poolout.shape[3] == switches.shape[3]):
raise ValueError('Mismatch in image shape.')
if not switches.shape[4] == 2:
raise ValueError('switches should only have length 2 in the 5. dimension.')
i, c, y, x, y_out, x_out = 0,0,0,0,0,0
y_min, y_max, x_min, x_max = 0,0,0,0
img_y, img_x = 0,0
img_y_max = 0
img_x_max = 0
value, new_value = 0,0
for i in range(n_imgs):
for c in range(n_channels):
for y_out in range(out_h):
y = y_out*stride_y
y_min = int_max(y-pool_h_top, 0)
y_max = int_min(y+pool_h_bottom, img_h)
for x_out in range(out_w):
x = x_out*stride_x
x_min = int_max(x-pool_w_left, 0)
x_max = int_min(x+pool_w_right, img_w)
value = -9e99
for img_y in range(y_min, y_max):
for img_x in range(x_min, x_max):
new_value = imgs[i, c, img_y, img_x]
if new_value > value:
value = new_value
img_y_max = img_y
img_x_max = img_x
poolout[i, c, y_out, x_out] = value
switches[i, c, y_out, x_out, 0] = img_y_max
switches[i, c, y_out, x_out, 1] = img_x_max
def test_from_array(self, backend_config):
arr = backend_config.get_array(numpy.ndarray((2,), numpy.float32))
device = backend.GpuDevice.from_array(arr)
assert device is None
rospy.init_node('shadow_tc_learn_to_pick_ball_qlearn',
anonymous=True,
log_level=rospy.WARN)
# Create the Gym environment
env = gym.make('ShadowTcGetBall-v0')
rospy.loginfo("Gym environment done")
# Set the logging system
rospack = rospkg.RosPack()
pkg_path = rospack.get_path('my_shadow_tc_openai_example')
outdir = pkg_path + '/training_results'
env = wrappers.Monitor(env, outdir, force=True)
rospy.loginfo("Monitor Wrapper started")
last_time_steps = numpy.ndarray(0)
# Loads parameters from the ROS param server
# Parameters are stored in a yaml file inside the config directory
# They are loaded at runtime by the launch file
Alpha = rospy.get_param("/shadow_tc/alpha") # 0.1
Epsilon = rospy.get_param("/shadow_tc/epsilon") # 0.9
Gamma = rospy.get_param("/shadow_tc/gamma") # 0.7
epsilon_discount = rospy.get_param("/shadow_tc/epsilon_discount")
nepisodes = rospy.get_param("/shadow_tc/nepisodes") # 500
nsteps = rospy.get_param("/shadow_tc/nsteps") #10000
# Initialises the algorithm that we are going to use for learning
qlearn = DQN(
N_ACTIONS=env.action_space.n,
N_STATES=env.observation_space.shape[0]) # actions = [0, 1, ... ,7]
def setUp(self):
# Example 12.1.5 from Fletcher
v = lambda x: -x[0, 0] - x[1, 0]
del_v = lambda x: np.ndarray([[-1, -1]])
c = [lambda x: 1 - x[0, 0]**2 - x[1, 0]**2]
self.p = opt.Problem(v, eq_const=c)
