def __call__(self, variables):
in_audio, curr_step, nx_step = variables
batchsize = in_audio.shape[0]
sequence = in_audio.shape[1]
self.loss = loss = 0
state = self.state
self.ot = xp.std(curr_step, axis=1)
for i in range(sequence):
h, state, y = self.forward(state, curr_step,
self.audiofeat(in_audio[:, i]), True)
loss += F.mean_squared_error(nx_step[:, i], y)
curr_step = y
ot = xp.std(nx_step[:, i], axis=1) * batchsize # y
delta_sgn = xp.sign(ot - self.ot)
if i > 0:
labels = xp.minimum(delta_sgn + self.delta_sgn, 1)
labels = xp.asarray(labels, dtype=xp.int32)
loss2 = F.contrastive(
h, self.h, labels
) / sequence # .mean_squared_error mean_absolute_error
if float(chainer.cuda.to_cpu(loss2.data)) > 0.:
loss += loss2 # F.mean_squared_error mean_absolute_error
self.h = h
self.ot = ot
self.delta_sgn = delta_sgn
self.loss = loss
stdout.write('loss={:.04f}\r'.format(
float(chainer.cuda.to_cpu(loss.data))))
stdout.flush()
return self.loss
def _percentiles(data, sigma=(0.13, 2.28, 15.87, 50, 84.13, 97.72, 99.87)):
"""Compute percentiles for data and return an array with the same length
as the number of elements in ``sigma``.
Args:
data (array): 1-dimensional NumPy or CuPy arryay.
sigma (tuple): Sigmas for which percentiles are computed. Defaults to
the three-sigma-rule. See: https://en.wikipedia.org/wiki/Percentile
Returns:
array: Array of percentiles.
"""
# TODO: Make percentile computation faster for GPUs.
# To CPU before computing the percentiles since CuPy doesn't implement
# np.percentile().
if cupy.get_array_module(data) is cupy:
data = cupy.asnumpy(data)
try:
ps = np.percentile(data, sigma)
except IndexError:
# If data is missing from uninitialized parameters, add
# NaN placeholders instead of skipping the measurements completely
# or registering zeros.
ps = np.array((float('NaN'), ) * 7)
# Back to GPU when percentiles are computed.
if cupy.get_array_module(data) is cupy:
ps = cupy.asarray(ps)
return ps
def img_tuple(state,action,Reward,state_dash,episode_end):
_state=""
_action=cp.zeros(1, dtype=cp.uint8)
_Reward=np.zeros((1, 1), dtype=np.float32)
_state_dash=""
_episode_end=np.zeros((1, 1), dtype=np.bool)
_state=state
_action[0]=cp.asarray(action)
_Reward[0][0]=Reward
_state_dash=state_dash
_episode_end[0][0]=episode_end
return (_state,_action,_Reward,_state_dash,_episode_end)
def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
scales=2**xp.arange(3, 6)):
"""
Generate anchor (reference) windows by enumerating aspect ratios X
scales wrt a reference (0, 0, 15, 15) window.
"""
base_anchor = xp.array([1, 1, base_size, base_size]) - 1
ratio_anchors = _ratio_enum(base_anchor, xp.asarray(ratios))
anchors = xp.vstack([_scale_enum(ratio_anchors[i, :], scales)
for i in range(ratio_anchors.shape[0])])
return anchors
def train_VAE(
gpu=GPU,
dataset_fileName=f'{DATASET_SAVE_PATH}/wsj0_normalize_{N_FFT}_{HOP_LENGTH}.pic'
):
file_suffix = f"normal-scale=gamma-D={N_LATENT}"
if os.path.isfile(MODEL_SAVE_PATH +
'/model-best-{0}.npz'.format(file_suffix)):
print(f"{MODEL_SAVE_PATH}model-best-{file_suffix}.npz already exist")
exit
cuda.get_device_from_id(gpu).use()
# Load dataset
with open(dataset_fileName, 'rb') as f:
dataset = pic.load(f)
n_data = dataset.shape[1]
# Prepare VAE model
model = network_VAE.VAE(n_freq=int(N_FFT / 2 + 1), n_latent=N_LATENT)
model.to_gpu()
# Setup Optimizer
optimizer = optimizers.Adam(LEARNING_RATE)
optimizer.setup(model)
# Learning loop
min_loss = np.inf
loss_list = []
for epoch in range(N_EPOCH):
print('Epoch:', epoch + 1)
sum_loss = 0
perm = np.random.permutation(n_data)
for ii in progressbar(range(0, n_data, BATCH_SIZE)):
minibatch = dataset[:, perm[ii:ii + BATCH_SIZE]].T
scales = np.random.gamma(2, 0.5, (len(minibatch)))
minibatch = minibatch * scales[:, None]
x = chainer.Variable(cp.asarray(minibatch, dtype=cp.float32))
optimizer.update(model.get_loss_func(), x)
sum_loss += float(model.loss.data) * BATCH_SIZE
loss_list.append(float(model.loss.data))
sum_loss /= n_data
print("Loss:", sum_loss)
print('save the model and optimizer')
serializers.save_npz(
MODEL_SAVE_PATH + 'model-{0}.npz'.format(file_suffix), model)
with open(MODEL_SAVE_PATH + 'loss-{0}.pic'.format(file_suffix),
'wb') as f:
pic.dump(loss_list, f)
if sum_loss < min_loss:
shutil.copyfile(
MODEL_SAVE_PATH + 'model-{0}.npz'.format(file_suffix),
MODEL_SAVE_PATH + 'model-best-{0}.npz'.format(file_suffix))
min_loss = sum_loss
sum_loss = 0
def handle(self):
save_img=0
f=open(args.model,'rb')
model = pickle.load(f)
f.close()
cuda.get_device(args.gpu).use()
model.to_gpu()
if os.path.isfile(args.model2):
f=open(args.model2,'rb')
model2 = DQN.DQN_class()
model2.model = pickle.load(f)
model2.target_model_update()
f.close()
# cuda.get_device(args.gpu2).use()
model2.model.to_gpu()
model2.model_target.to_gpu()
else:
model2 = DQN.DQN_class()
# cuda.get_device(args.gpu2).use()
model2.model.to_gpu()
model2.model_target.to_gpu()
que_img = []
que_img_pass = []
#train = open('train.txt','w')
episodeCounter=0
print "episode "+str(episodeCounter+1)
double_error=0
linelist=["python compute_mean.py train.txt","python train_imagenet.py -g 0 -B 16 -b 1 -E 20 train.txt test.txt 2>&1 | tee log"]
eps=1
base_time=time.time()
time_bool=True
try:
while True:
self.data = self.request.recv(1024).strip()
if time_bool:
time_bool=False
base_time=base_time-(base_time-time.time())
self.request.send("receive")
buf=''
recvlen=0
while recvlen<int(self.data):
receivedstr=self.request.recv(1024*1024)
recvlen+=len(receivedstr)
buf +=receivedstr
img=np.fromstring(buf,dtype='uint8').reshape(740-290,670-284,3)
img_check=img[464-290:494-290,284-284:382-284,:].astype(np.float32)
_img_check = img_check.transpose(2,0,1) - mean_image
#rsvmsg=cv2.imdecode(rsvmsg,1)
height, width, depth = img.shape
new_height = size
new_width = size
if height > width:
new_width = size * width / height
else:
new_height = size * height / width
crop_height_start = ( size - new_height ) / 2
crop_height_end = crop_height_start + new_height
crop_width_start = ( size - new_width) / 2
crop_width_end = crop_width_start + new_width
resized_img = cv2.resize(img, (new_width, new_height))
cropped_img = np.zeros((size,size,3),np.uint8)
#cropped_img.fill(255) #white ver
cropped_img[crop_height_start:crop_height_end,crop_width_start:crop_width_end] = resized_img
_cropped_img = cropped_img.astype(np.float32).transpose(2,0,1)
_cropped_img -= mean_image2
_cropped_img /= 255
state_dash = cp.ndarray((1, 3, size, size), dtype=cp.float32)
state_check = cp.ndarray((1, 3, 30, 98), dtype=cp.float32)
state_dash[0]=cp.asarray(_cropped_img)
state_check[0]=cp.asarray(_img_check)
action_dash = model2.e_greedy(state_dash, eps)
_name=np.argmax(model.predict(state_check,train=False).data.get())
print action_dash
state_dash_path="save/img"+str(save_img).zfill(7)+".jpg"
cv2.imwrite(state_dash_path,cropped_img)
print "saved "+state_dash_path
save_img+=1
if len(que_img_pass)>0:
state,action=que_img_pass.pop(0)
Reward=(time.time()-base_time)*(1.0/10**3)
episode_end=False
if _name==-1:
Reward=-1
episode_end=True
que_img.append(img_tuple(state,action,Reward,state_dash_path,episode_end))
print Reward
if _name==1:
self.request.send(str(-2))
self.request.recv(1024)
if episodeCounter==19:
while len(que_img)>100000:
que_img.pop(0)
eps=1
DQN_batch_size=1
_state=cp.zeros((DQN_batch_size, 3, 224, 224),dtype=cp.float32)
_action=cp.zeros(DQN_batch_size, dtype=cp.uint8)
_Reward=np.zeros((DQN_batch_size, 1), dtype=np.float32)
_state_dash=cp.zeros((DQN_batch_size, 3, 224, 224), dtype=cp.float32)
_episode_end=np.zeros((DQN_batch_size, 1), dtype=np.bool)
for epoch in range(5):
print "epoch"+str(epoch+1)
counter=0
perm = np.random.permutation(len(que_img))
for idx in perm:
s_replay,a_replay,r_replay,s_dash_replay,episode_end_replay=que_img[idx]
_state[counter:counter+1]=load_img(s_replay)
_action[counter:counter+1]=a_replay
_Reward[counter:counter+1]=r_replay
_state_dash[counter:counter+1]=load_img(s_dash_replay)
_episode_end[counter:counter+1]=episode_end_replay
if counter==DQN_batch_size-1:
counter=0
model2.optimizer.zero_grads()
loss, _ = model2.forward(_state,_action,_Reward,_state_dash,_episode_end)
loss.backward()
model2.optimizer.update()
del loss
else:
counter+=1
del s_replay,a_replay,r_replay,s_dash_replay,episode_end_replay
episodeCounter=0
pickle.dump(model2.model, open(args.model2, 'wb'), -1)
model2.target_model_update()
#model2.model_target.to_gpu()
else:
episodeCounter+=1
base_time=time.time()
time_bool=True
print "episode "+str(episodeCounter+1)
self.request.send("OK")
else:
que_img_pass.append(img_pass_tuple(state_dash_path,action_dash))
self.request.send(str(action_dash))
eps -= 5.0/10**4
print eps
if eps < 0.1:
eps = 0.1
except KeyboardInterrupt:
pass
def load_img(path):
image = np.asarray(Image.open(path)).transpose(2, 0, 1).astype(np.float32)
image -= mean_image2
image /= 255
return cp.asarray(image.reshape(1,3,224,224))
def img_pass_tuple(state,action):
_state=""
_action=cp.zeros(1, dtype=cp.uint8)
_state=state
_action[0]=cp.asarray(action)
return (_state,_action)
def __call__(self, rpn_cls_prob, rpn_bbox_pred, im_info, train):
# Algorithm:
#
# for each (H, W) location i
# generate A anchor boxes centered on cell i
# apply predicted bbox deltas at cell i to each of the A anchors
# clip predicted boxes to image
# remove predicted boxes with either height or width < threshold
# sort all (proposal, score) pairs by score from highest to lowest
# take top pre_nms_topN proposals before NMS
# apply NMS with threshold 0.7 to remaining proposals
# take after_nms_topN proposals after NMS
# return the top proposals (-> RoIs top, scores top)
pre_nms_topN = self.RPN_PRE_NMS_TOP_N if train else 6000
post_nms_topN = self.RPN_POST_NMS_TOP_N if train else 300
nms_thresh = self.RPN_NMS_THRESH
min_size = self.RPN_MIN_SIZE
# the first set of _num_anchors channels are bg probs
# the second set are the fg probs, which we want
scores = rpn_cls_prob.data[:, self._num_anchors:, :, :]
bbox_deltas = rpn_bbox_pred.data
im_info = im_info[0, :]
# 1. Generate proposals from bbox deltas and shifted anchors
height, width = scores.shape[-2:]
# Enumerate all shifts
shift_x = np.arange(0, width) * self._feat_stride
shift_y = np.arange(0, height) * self._feat_stride
shift_x, shift_y = xp.asarray(np.meshgrid(shift_x, shift_y))
shifts = xp.vstack((shift_x.ravel(), shift_y.ravel(),
shift_x.ravel(), shift_y.ravel())).transpose()
# Enumerate all shifted anchors:
#
# add A anchors (1, A, 4) to
# cell K shifts (K, 1, 4) to get
# shift anchors (K, A, 4)
# reshape to (K*A, 4) shifted anchors
A = self._num_anchors
K = shifts.shape[0]
anchors = self._anchors.reshape((1, A, 4)) + \
shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4))
# Transpose and reshape predicted bbox transformations to get them
# into the same order as the anchors:
#
# bbox deltas will be (1, 4 * A, H, W) format
# transpose to (1, H, W, 4 * A)
# reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a)
# in slowest to fastest order
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
# Same story for the scores:
#
# scores are (1, A, H, W) format
# transpose to (1, H, W, A)
# reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a)
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# Convert anchors into proposals via bbox transformations
proposals = bbox_transform_inv(anchors, bbox_deltas)
# 2. clip predicted boxes to image
proposals = clip_boxes(proposals, im_info[:2])
# 3. remove predicted boxes with either height or width < threshold
# (NOTE: convert min_size to input image scale stored in im_info[2])
if chainer.cuda.available:
proposals = xp.asnumpy(proposals)
scores = xp.asnumpy(scores)
keep = _filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# 4. sort all (proposal, score) pairs by score from highest to lowest
# 5. take top pre_nms_topN (e.g. 6000)
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]
# Output rois blob
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
batch_inds = np.zeros((proposals.shape[0], 1), dtype=xp.float32)
rois = xp.asarray(np.hstack((batch_inds, proposals)), dtype=xp.float32)
return rois
def __init__(self, feat_stride=16, anchor_scales=[4, 8, 16, 32]):
self._feat_stride = feat_stride
self._anchors = xp.asarray(
generate_anchors(scales=xp.array(anchor_scales)))
self._num_anchors = self._anchors.shape[0]
