def __init__(self, feature_list, std):
self.matrix_size = (15, 20)
self.backgrounder = Backgrounder(std, self.matrix_size)
self.curr_off_pixels = []
self.curr_object_pixels = []
self.curr_color = cbf.get_emotion_color_by_angle(60)
self.hex_array = cbf.to_hex_array(cbf.gaussian_color_matrix(self.curr_color, std=std, size=self.matrix_size))
self.enabled_features = {}
self.timer = TimeQuantizer()
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(config_file)
self.numpixels = 300
self.strip = TestStrip(numpixels=self.numpixels)
self.magnituder = FluxMagnituder(self.strip)
self.rasta_shower = CoefficientShower(self.strip, len(smile_rastas))
rec6 = Rectangle(visualizer=self, coordinate=(10, 6), size=(3, 3), led_strip=self.strip, color=yellow)
self.objects = [rec6]
for key in config['features']:
if int(config['features'][key]) == 1 and key in feature_list:
self.enabled_features[key] = feature_list.index(key)
print('Enabled Features: ', self.enabled_features)
self.feature_maxima = np.zeros(len(feature_list))
def __init__(self, proto):
self._proto = proto
self._p = Predictor(model_a, model_v, batch_size=1)
self._paused = False
self.smile_extract = None
self.llds = Queue()
self.funcs = Queue()
self.arousal = Queue()
self.valence = Queue()
self.visualizer = None
self.timer = TimeQuantizer()
def __init__(self,
feature_list,
std,
led_strip,
vis_type=VisualizerTypes.Rasta):
self.matrix_size = (15, 20)
self.backgrounder = Backgrounder(std, self.matrix_size)
self.curr_off_pixels = []
self.curr_object_pixels = []
self.type = vis_type
self.curr_color = cbf.get_emotion_color_by_angle(60)
self.hex_array = cbf.to_hex_array(
cbf.gaussian_color_matrix(self.curr_color,
std=std,
size=self.matrix_size))
self.timer = TimeQuantizer()
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(config_file)
self.numpixels = 300
self.strip = led_strip
self.rec = Rectangle(visualizer=self,
coordinate=(10, 6),
size=(3, 3),
led_strip=self.strip,
color=yellow)
if self.type == VisualizerTypes.BouncingSquare:
self.objects = [self.rec]
self.enabled_features = EnabledFeatures(config, feature_list)
print('Enabled Features: ', self.enabled_features.list())
self.feature_maxima = np.zeros(len(feature_list))
self.rasta_shower = CoefficientShower(self.strip, len(HLDs.rastas))
if self.type == VisualizerTypes.Rasta:
self.objects = [self.rasta_shower]
# if self.type == VisualizerTypes.Magnituder:
self.magnituder = FluxMagnituder(self.strip)
def main(_):
smile_extract = subprocess.Popen([SMILExtract, '-C', smile_config],
bufsize=1,
stdout=subprocess.PIPE)
# the first to lines are the names of the features
lld_list = lh.make_feature_list_from_smileout(
smile_extract.stdout.readline())
print(lld_list)
func_list = lh.make_feature_list_from_smileout(
smile_extract.stdout.readline())
assert (len(lld_list) < len(func_list)), 'Funcs initialized before LLDS'
# delay the audio output in order to sync audio and visuals
subprocess.run(['pacmd', 'unload-module module-loopback'], check=True)
subprocess.run(['pacmd', 'load-module module-loopback latency_msec=615'],
check=True)
llds = Queue()
funcs = Queue()
StreamReader(smile_extract.stdout, llds, funcs, len(lld_list))
timer = TimeQuantizer()
visualizer = Visualizer(lld_list, std=0.1)
visualizer.update_base_color(np.random.rand(), np.random.rand())
while True:
timer.reset()
if not llds.empty():
visualizer.update_visuals(llds.get())
print('Total Loop Time: ', timer.measure_total(), 'Queue Size: ',
llds.qsize())
if llds.qsize() > 10:
for _ in range(5):
llds.get()
elif not funcs.empty():
funcs.get()
visualizer.update_base_color(np.random.rand() - 0.5,
np.random.rand() - 0.5)
print('Updating base color time: ', timer.measure_step())
else:
print('Sleeping')
time.sleep(0.003)
class Producer(object):
def __init__(self, proto):
self._proto = proto
self._p = Predictor(model_a, model_v, batch_size=1)
self._paused = False
self.smile_extract = None
self.llds = Queue()
self.funcs = Queue()
self.arousal = Queue()
self.valence = Queue()
self.visualizer = None
self.timer = TimeQuantizer()
def resumeProducing(self):
self._paused = False
if self.smile_extract is None:
self.smile_extract = subprocess.Popen(
[SMILExtract, '-C', smile_config], stdout=subprocess.PIPE)
# the first two lines are the names of the features
lld_list = lh.make_feature_list_from_smileout(
self.smile_extract.stdout.readline())
func_list = lh.make_feature_list_from_smileout(
self.smile_extract.stdout.readline())
# has never been thrown yet
assert (len(lld_list) <
len(func_list)), 'Funcs initialized before LLDS'
StreamReader(self.smile_extract.stdout, self.llds, self.funcs,
len(lld_list))
self.visualizer = Visualizer(lld_list,
std=0.2,
led_strip=TcpStrip(self._proto))
self.visualizer.update_base_color(np.random.rand(),
np.random.rand())
self._p.start_predicting(self.funcs, self.arousal, self.valence)
while not self._paused:
if self.timer.measure_total() > 60:
self.timer.reset()
time.sleep(0.5)
self.reduce_queue_size(self.llds)
if not self.llds.empty():
# print('LLds queue size', llds.qsize())
self.reduce_queue_size(self.llds)
self.visualizer.update_visuals(self.llds.get())
if not self.arousal.empty() and not self.valence.empty():
self.reduce_queue_size(self.arousal)
self.reduce_queue_size(self.valence)
if self.timer.measure_tick() > 5:
self.timer.set_tick()
update_base_color_counter = 0
a = np.float(self.arousal.get() / 1000)
v = np.float(self.valence.get() / 1000)
# print('Arousal: {}, Valence: {}'.format(a, v))
self.visualizer.update_base_color(a, v)
else:
time.sleep(0.005)
@staticmethod
def reduce_queue_size(queue):
if queue.qsize() > 5:
for _ in range(queue.qsize() - 1):
queue.get()
def pauseProducing(self):
self._paused = True
def stopProducing(self):
if self.smile_extract is not None:
self.smile_extract.kill()
log.msg('Stop Producing')
self._proto.transport.unregisterProducer()
self._proto.transport.loseConnection()
class Visualizer:
def __init__(self, feature_list, std):
self.matrix_size = (15, 20)
self.backgrounder = Backgrounder(std, self.matrix_size)
self.curr_off_pixels = []
self.curr_object_pixels = []
self.curr_color = cbf.get_emotion_color_by_angle(60)
self.hex_array = cbf.to_hex_array(cbf.gaussian_color_matrix(self.curr_color, std=std, size=self.matrix_size))
self.enabled_features = {}
self.timer = TimeQuantizer()
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(config_file)
self.numpixels = 300
self.strip = TestStrip(numpixels=self.numpixels)
self.magnituder = FluxMagnituder(self.strip)
self.rasta_shower = CoefficientShower(self.strip, len(smile_rastas))
rec6 = Rectangle(visualizer=self, coordinate=(10, 6), size=(3, 3), led_strip=self.strip, color=yellow)
self.objects = [rec6]
for key in config['features']:
if int(config['features'][key]) == 1 and key in feature_list:
self.enabled_features[key] = feature_list.index(key)
print('Enabled Features: ', self.enabled_features)
self.feature_maxima = np.zeros(len(feature_list))
def update_visuals(self, llds):
self.timer.reset()
self.feature_maxima = np.maximum(self.feature_maxima, llds)
self.feature_maxima = self.feature_maxima * 0.999
if smile_flux in self.enabled_features:
flux, centroid, rms, entropy, flux_max, energy_max, \
energy_delta, spec_rolloff, hnr = self._get_features(llds)
self.update_palette(centroid, rms, hnr)
if (flux > flux_max / 3) and (energy_delta > 0.007):
for rec in self.objects:
rec.random_move(flux, flux_max, flush=True)
self.strip.show()
def update_base_color(self, arousal, valence):
self.curr_color = cbf.get_emotion_color(arousal, valence)
self.backgrounder.update_gaussian_mask(arousal, valence)
for o in self.objects:
o.update_color(self.curr_color)
def update_palette(self, centroid, rms, entropy):
self.hex_array, off_pixels = self.backgrounder.modulate_color(self.curr_color, centroid, rms, entropy)
self.draw_whole_background(off_pixels)
self.timer.reset()
for o in self.objects:
o.redraw()
def _get_features(self, llds):
print(llds)
flux = llds[self.enabled_features[smile_flux]]
centroid = llds[self.enabled_features[smile_centroid]]
rms = llds[self.enabled_features[smile_rms]]
entropy = llds[self.enabled_features[smile_entropy]]
flux_max = self.feature_maxima[self.enabled_features[smile_flux]]
energy_max = self.feature_maxima[self.enabled_features[smile_rms]]
energy_delta = llds[self.enabled_features[smile_delta_rms]]
spect_rolloff = llds[self.enabled_features[smile_rolloff]]
# hnr = llds[self.enabled_features[smile_hnr]]
spect_harm = llds[self.enabled_features[smile_harmonicity]]
return flux, centroid, rms, entropy, flux_max, energy_max, energy_delta, spect_rolloff, spect_harm
def _get_mfccs(self, llds):
mfccs = [llds[self.enabled_features[mf]] for mf in smile_mfccs]
return mfccs
def _get_rastas(self, llds):
rastas = [llds[self.enabled_features[ra]] for ra in smile_rastas]
return rastas
def _draw_all(self, hex_array):
i = 0
for color in hex_array:
self.strip.setPixelColor(i, color)
i += 1
self.strip.show()
def draw_whole_background(self, off_pixels=None):
non_background_pixels = []
object_pixels = []
for o in self.objects:
object_pixels += o.get_object_pixels()
if off_pixels is not None:
for i in off_pixels:
if i not in object_pixels:
self.strip.setPixelColor(i, black)
non_background_pixels += off_pixels
non_background_pixels += object_pixels
self.curr_off_pixels = off_pixels
self.curr_object_pixels = object_pixels
i = 0
if len(non_background_pixels) > 150:
background_pixels = set(range(300)).difference(set(non_background_pixels))
for color in self.hex_array:
if i in background_pixels:
self.strip.setPixelColor(i, color)
i += 1
else:
for color in self.hex_array:
if i not in non_background_pixels:
self.strip.setPixelColor(i, color)
i += 1
def draw_pixels(self, pixels):
i = 0
object_pixels = []
for o in self.objects:
object_pixels += o.get_object_pixels()
for color in self.hex_array:
if i in pixels:
if i in self.curr_off_pixels:
self.strip.setPixelColor(i, black)
elif i not in self.curr_object_pixels:
self.strip.setPixelColor(i, color)
else:
for o in self.objects:
if i in o.get_object_pixels():
self.strip.setPixelColor(i, o.color)
i += 1
self.strip.show()
def train(state_size, num_layers, batch_size, num_outputs, sequence_length, affect_type, std):
######################################################
# Preliminary
######################################################
m_list = ['A', 'V'] # define if the model is for arousal or valence
config_str = 'egemaps_{}_size{}_step{}_bs{}_sl{}_nl{}_ss{}_no{}_std{}'.format(m_list[affect_type],
framesize,
framestep,
batch_size,
sequence_length,
num_layers,
state_size,
num_outputs,
std)
rnn_config = RnnConfig(state_size=state_size,
num_layers=num_layers,
num_outputs=num_outputs)
# Remove possible older graphs
tf.reset_default_graph()
# data = Dataset(all_feature_sets, filenames=files)
# _, test_files = data.split(batch_size, sequence_length, test_size)
data = MirexDataSplit(train_sets, test_sets, batch_size, sequence_length)
mean_arousal, mean_valence = data.get_train_labels_mean()
# Create placeholders: use None instead of batch_size to enable a variable batch size for testing
x = tf.placeholder(tf.float32, [None, None, data.train.num_features], name='x')
y = tf.placeholder(tf.float32, [None, None, 2], name='y')
mode = tf.placeholder(tf.string, name='mode')
tf_mean_list, tf_std_list = tf.init_ops.mean_std_placeholders()
########################################################
# Define Tensorflow operations aka graph nodes
########################################################
with tf.name_scope('gaussian_input_layer'):
x = tf.cond(tf.equal(mode, 'train'), lambda: gaussian_noise_layer(x, std), lambda: x)
with tf.variable_scope('model_{}'.format(m_list[affect_type])):
model = TrainModel(x, [num_layers, 2, None, state_size], rnn_config)
with tf.name_scope('error'):
rmse = metrics.l1diff_rms_error(model.pred, y[:, :, affect_type])
with tf.name_scope('adam_optimizer'):
optimizer = tf.init_ops.adam_optimizer(rmse, data.train.num_batches * data.train.num_sequences)
with tf.name_scope('feature_means_std'):
tf_means, tf_std = tf.init_ops.mean_std_variable(data.train.num_features)
mean_op = tf_means.assign(tf.reduce_mean(tf.stack(tf_mean_list), axis=0, name='mean_op'))
std_op = tf_std.assign(tf.reduce_mean(tf.stack(tf_std_list), axis=0, name='std_op'))
############################################################################################################
# Training & Evaluation
############################################################################################################
# Enable flexible GPU memory allocation
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
timer = TimeQuantizer()
with tf.Session(config=config) as sess:
# writer enables TensorBoard visualization of the graph
# Command: tensorboard --logdir=./graphs
writer = tf.summary.FileWriter('./graphs', sess.graph)
sess.run(tf.global_variables_initializer())
# plotter = MyPlotter()
best_result = 180
feature_means = []
feature_vars = []
metrix = metrics.MetricsContainer(filenames=test_files)
metrix.save_filenames('../metrics/{}/'.format(config_str))
c_state = np.zeros((num_layers, 2, batch_size, state_size))
for epoch in range(num_epochs):
epoch_loss = 0
data.train.shuffle()
for i in range(0, data.train.num_batches):
data.train.next_batch()
mean, var = data.train.normalize_mode_train()
feature_means.append(mean)
feature_vars.append(var)
for seq_x, seq_y in data.train.sequences:
# print('Seq_x shape:', seq_x.shape, 'Seq_y shape:', seq_y.shape)
_, c_state, rms = \
sess.run([optimizer, model.state, rmse],
feed_dict={x: seq_x,
y: seq_y,
model.init_state: c_state,
mode: 'train'})
epoch_loss += rms
metrix.train_loss.append(epoch_loss)
############################################################################################################
# Evaluate the model_type regularly
############################################################################################################
if (epoch + 1) % validation_epochs == 0:
print('Training time after {} epochs: {} minutes\n'.format(
epoch + 1, round(timer.measure_total() / 60, 3)))
random_loss = 0
test_loss = 0
test_pred_a = []
test_pred_v = []
test_lab_a = []
test_lab_v = []
sess.run([mean_op, std_op], feed_dict={tf_mean_list: feature_means, tf_std_list: feature_vars})
# print('Feature means:', tf_means.eval())
c_state = np.zeros((num_layers, 2, batch_size, state_size))
for i in range(0, data.test.num_batches):
data.test.next_batch()
data.test.normalize_mode_test(tf_means.eval(), tf_std.eval())
for seq_x, seq_y in data.test.sequences:
[c_state, rms, p] = \
sess.run([model.state, rmse, model.pred],
feed_dict={x: seq_x,
y: seq_y,
model.init_state: c_state,
mode: 'test'})
random_loss += np.sqrt(np.mean(np.square(mean_arousal - seq_y[:, :, affect_type])))
test_loss += rms
metrix.extend_predictions_labels(p, seq_y[:, :, affect_type])
metrix.test_loss.append(test_loss)
tot_sequences = data.test.num_sequences * data.test.num_batches
test_rmse = test_loss / tot_sequences
random_rmse = random_loss / tot_sequences
metrix.save_predictions_labels('../metrics/{}/'.format(config_str), epoch+1)
metrix.flush_predictions_labels()
print('{} Epoch {}: Test RMSE: {}, Random RMSE: {}'.format(
config_str, epoch + 1, test_rmse, random_rmse))
if test_rmse < best_result:
best_result = test_rmse
result_str = 'Epoch {}: RMS: {}\n'.format(epoch + 1, test_rmse)
save_results(config_str, result_str)
saver = tf.train.Saver()
saver.save(sess, 'model/' + config_str + '/model.ckpt', global_step=epoch + 1)
metrix.save_train_test_loss('../metrics/{}/'.format(config_str))
writer.close()
class Visualizer:
def __init__(self,
feature_list,
std,
led_strip,
vis_type=VisualizerTypes.Rasta):
self.matrix_size = (15, 20)
self.backgrounder = Backgrounder(std, self.matrix_size)
self.curr_off_pixels = []
self.curr_object_pixels = []
self.type = vis_type
self.curr_color = cbf.get_emotion_color_by_angle(60)
self.hex_array = cbf.to_hex_array(
cbf.gaussian_color_matrix(self.curr_color,
std=std,
size=self.matrix_size))
self.timer = TimeQuantizer()
config = configparser.ConfigParser(allow_no_value=True)
config.optionxform = str
config.read(config_file)
self.numpixels = 300
self.strip = led_strip
self.rec = Rectangle(visualizer=self,
coordinate=(10, 6),
size=(3, 3),
led_strip=self.strip,
color=yellow)
if self.type == VisualizerTypes.BouncingSquare:
self.objects = [self.rec]
self.enabled_features = EnabledFeatures(config, feature_list)
print('Enabled Features: ', self.enabled_features.list())
self.feature_maxima = np.zeros(len(feature_list))
self.rasta_shower = CoefficientShower(self.strip, len(HLDs.rastas))
if self.type == VisualizerTypes.Rasta:
self.objects = [self.rasta_shower]
# if self.type == VisualizerTypes.Magnituder:
self.magnituder = FluxMagnituder(self.strip)
def update_visuals(self, llds):
self.feature_maxima = np.maximum(self.feature_maxima, llds)
self.feature_maxima = self.feature_maxima * 0.999
if self.type == VisualizerTypes.Rasta:
self.update_rastas(llds)
flux, centroid, rms, entropy, flux_max, energy_max, \
energy_delta, spec_rolloff, hnr = self.enabled_features.get_features(self.feature_maxima, llds)
self.update_palette(centroid, rms, hnr)
if should_trigger_movement(flux, flux_max, energy_delta):
if self.timer.measure_total() > 5 * 60:
print('updating visuals')
self.next_visualizer_type()
self.timer.reset()
if self.is_bouncing_squares():
if self.timer.measure_tick() > 10:
self.initiate_switch_bounce(flux, flux_max)
else:
for rec in self.objects:
rec.random_move(flux, flux_max, flush=True)
if self.type == VisualizerTypes.Magnituder:
self.magnituder.show(self.hex_array, flux, flux_max)
#
# # print('updating visuals time: ', self.timer.measure_total())
#
self.strip.show()
def update_rastas(self, llds):
rastas = self.enabled_features.get_rastas(llds)
self.rasta_shower.show(rastas)
def update_base_color(self, arousal, valence):
self.curr_color = cbf.get_emotion_color(arousal, valence)
self.backgrounder.update_gaussian_mask(arousal, valence)
for o in self.objects:
o.update_color(self.curr_color)
def update_palette(self, centroid, rms, entropy):
self.hex_array, self.curr_off_pixels = self.backgrounder.modulate_color(
self.curr_color, centroid, rms, entropy)
# if self.type == VisualizerTypes.BouncingSquare or self.type == VisualizerTypes.MultiBouncingSquare:
# self.curr_off_pixels = set(range(300))
self.curr_object_pixels = self.get_object_pixels()
self.draw_whole_background(self.curr_object_pixels,
self.curr_off_pixels)
for o in self.objects:
o.redraw()
def _draw_all(self, hex_array):
i = 0
for color in hex_array:
self.strip.setPixelColor(i, color)
i += 1
self.strip.show()
def draw_whole_background(self, object_pixels, off_pixels):
if off_pixels is not None:
self.set_off_pixels_black(object_pixels, off_pixels)
non_background_pixels = []
non_background_pixels += object_pixels
non_background_pixels += off_pixels
self.draw_background(non_background_pixels)
def get_object_pixels(self):
object_pixels = []
for o in self.objects:
object_pixels += o.get_object_pixels()
return object_pixels
def draw_background(self, non_background_pixels):
i = 0
if len(non_background_pixels) > 150:
background_pixels = set(range(300)).difference(
set(non_background_pixels))
for color in self.hex_array:
if i in background_pixels:
self.strip.setPixelColor(i, color)
i += 1
else:
for color in self.hex_array:
if i not in non_background_pixels:
self.strip.setPixelColor(i, color)
i += 1
def set_off_pixels_black(self, object_pixels, off_pixels):
for i in off_pixels:
if i not in object_pixels:
self.strip.setPixelColor(i, black)
def draw_pixels(self, pixels):
i = 0
for color in self.hex_array:
if i in pixels:
if i in self.curr_off_pixels:
self.strip.setPixelColor(i, black)
elif i not in self.curr_object_pixels:
self.strip.setPixelColor(i, color)
else:
for o in self.objects:
if i in o.get_object_pixels():
self.strip.setPixelColor(i, o.color)
i += 1
self.strip.show()
def next_visualizer_type(self):
if self.is_bouncing_squares():
self.update_visualizer_type(VisualizerTypes.Magnituder)
elif self.type == VisualizerTypes.Magnituder:
self.update_visualizer_type(VisualizerTypes.Rasta)
else:
self.update_visualizer_type(VisualizerTypes.BouncingSquare)
def update_visualizer_type(self, new_type):
if new_type == VisualizerTypes.BouncingSquare:
self.type = VisualizerTypes.BouncingSquare
self.objects = [self.rec]
elif new_type == VisualizerTypes.MultiBouncingSquare:
self.type = VisualizerTypes.MultiBouncingSquare
self.objects = self.get_rectangle_array(self.rec.x, self.rec.y)
elif new_type == VisualizerTypes.Magnituder:
self.type = VisualizerTypes.Magnituder
self.objects = []
elif new_type == VisualizerTypes.Rasta:
self.type = VisualizerTypes.Rasta
self.objects = [self.rasta_shower]
else:
self.type = VisualizerTypes.BouncingSquare
self.objects = [self.rec]
def initiate_switch_bounce(self, flux, flux_max):
if self.type == VisualizerTypes.MultiBouncingSquare:
if self.all_objects_at(self.rec.x, self.rec.y):
self.update_visualizer_type(VisualizerTypes.BouncingSquare)
self.timer.set_tick()
else:
self.all_objects_approach(self.rec.x, self.rec.y, flux,
flux_max)
else:
self.update_visualizer_type(VisualizerTypes.MultiBouncingSquare)
self.timer.set_tick()
def is_bouncing_squares(self):
return self.type == VisualizerTypes.BouncingSquare or self.type == VisualizerTypes.MultiBouncingSquare
def all_objects_at(self, x, y):
for rec in self.objects:
if not rec.is_at(x, y):
return False
return True
def all_objects_approach(self, x, y, flux, flux_max):
[rec.approach(x, y, flux, flux_max) for rec in self.objects]
def get_rectangle_array(self, x, y):
return [
Rectangle(visualizer=self,
coordinate=(x, y),
size=(1, 1),
led_strip=self.strip,
color=yellow) for _ in range(20)
]