def execute(self, context):
if not context.object.tempUserData:
self.report({'ERROR'}, "There's no loaded data")
return {'CANCELLED'}
scn = context.scene
idx = scn.speech2anim_data.selected_training_video_index
addon_path = wops.clear(
bpy.utils.user_resource("SCRIPTS", "addons") + config.ADDON_PATH)
try:
item = scn.speech2anim_data.training_videos_list[idx]
except IndexError:
return {'CANCELLED'}
training_file = CSVFile()
training_file.from_file(addon_path + '/output/tempdata/' + item.name +
'/training_data.csv')
training_file = training_file.to_dict()
training_info = CSVFile()
training_info.from_file(addon_path + '/output/tempdata/' + item.name +
'/training_info.csv')
training_info_dict = training_info.to_dict()
for label_group in config.LABEL_GROUPS:
group_name = label_group['group_name']
#print(group_name)
new_group_labels = [0] * len(training_file[group_name])
for i, label_name in enumerate(label_group['label_names']):
#print(label_name, i)
for j, frame in enumerate(
training_info_dict[config.FRAME_INDEX_COLNAME]):
context.scene.frame_set(int(frame))
value = int(context.object.tempUserData[label_name])
#print(frame, value, j)
training_info_dict[label_name][j] = value
if value:
#print("add to training","row:", j,"label:", i)
new_group_labels[j] = i + 1
training_file[group_name] = new_group_labels
training_info.from_dict(training_info_dict)
training_info.saveAs(addon_path + '/output/tempdata/' + item.name +
'/training_info')
new_training_file = CSVFile()
new_training_file.from_dict(training_file)
new_training_file.saveAs(addon_path + '/output/tempdata/' + item.name +
'/training_data')
return {'FINISHED'}
def GenerateFinalTrainingFile(model_output):
ncols = 0
nrows = 0
final_file = ""
ngroups = len(config.LABEL_GROUPS)
###export csv
#"""
first = glob.glob(config.TEMPDATA_OUTPUT + '/*/training_data.csv')[0]
final_csv = CSVFile()
temp = CSVFile()
temp.from_file(first)
final_csv.setHeader(temp.getHeader())
#"""
###export csv---
current_file = CSVFile()
for filename in glob.glob(config.TEMPDATA_OUTPUT + '/*/training_data.csv'):
current_file.from_file(filename)
nrows += current_file.get_row_count()
ncols = len(current_file.getHeader())
for row in current_file.getRows():
#ecsv
final_csv.addRow(row)
#ecsv--
#last ngroups rows are labels, which are ints
for v in row[:-ngroups]:
floatval = "0 "
try:
floatval = "{0:.10f}".format(float(v)) + " "
except:
print("Warning: can't transform to float: ", v)
final_file += floatval
for group_label in row[-ngroups:]:
final_file += group_label + ' '
final_file += '\n'
mkdir(config.DATA_OUTPUT + '/training')
final_csv.saveAs(config.DATA_OUTPUT + "/training/final_training_data")
with open(config.DATA_OUTPUT + "/training/final_training_data.txt",
"w") as output:
output.write("0\n 0 \n {} {} {}\n".format(ncols, nrows, ngroups))
for name in utils.get_group_names(config.LABEL_GROUPS):
output.write(name + " ")
output.write(model_output + "\n")
output.write(final_file)
def DoPredict(input_model, filename):
"""
Generates predicted labels file.
returns path to the generated file
"""
name = w2p(filename).split('/')[-1:][0].split('.')[0]
temp_folder = config.TEMPDATA_OUTPUT + '/' + name
mkdir(temp_folder)
output_folder = config.DATA_OUTPUT + '/prediction/' + name
mkdir(output_folder)
# get opensmile data
opensmile_data = CSVFile()
opensmile_data.from_file(config.OPENSMILE_OUTPUT + '/' + name +
'/features.csv')
#create prediction input data
#option 1 prediction
result_file = '1\n'
#-1 because first col is not feature
result_file += str(len(opensmile_data.getHeader()) - 1) + '\n'
result_file += str(opensmile_data.get_row_count()) + '\n'
result_file += input_model + '\n'
for row in opensmile_data.getRows():
features = row[2:]
for v in features:
floatval = "0 "
try:
floatval = "{0:.10f}".format(float(v)) + " "
except:
print("Warning: can't transform to float: ", v)
result_file += floatval
result_file += '\n'
with open(temp_folder + '/prediction_data.txt', 'w') as output:
output.write(result_file)
#get prediction output
with open(temp_folder + '/prediction_data.txt') as data:
with open(output_folder + '/predicted_labels.csv', 'w') as output:
subprocess.call([config.MODEL_MANAGER], stdin=data, stdout=output)
return output_folder + '/predicted_labels.csv'
def GenerateFinalRegressionTrainingFile():
ncols = 0
nrows = 0
final_file = ""
###export csv
first = glob.glob(config.TEMPDATA_OUTPUT +
'/*/regression_training_data.csv')[0]
final_csv = CSVFile()
temp = CSVFile()
temp.from_file(first)
final_csv.setHeader(temp.getHeader())
###export csv---
current_file = CSVFile()
for filename in glob.glob(config.TEMPDATA_OUTPUT +
'/*/regression_training_data.csv'):
current_file.from_file(filename)
nrows += current_file.get_row_count()
ncols = len(current_file.getHeader())
for row in current_file.getRows():
#ecsv
final_csv.addRow(row)
#ecsv--
for v in row:
final_file += "{0:.10f}".format(float(v)) + " "
final_file += "\n"
final_csv.saveAs(config.DATA_OUTPUT +
"/training/final_regression_training_data")
with open(
config.DATA_OUTPUT +
"/training/final_regression_training_data.txt", "w") as output:
outputDimensions = len(config.WINDOW_VALUES)
inputDimensions = ncols - outputDimensions
output.write("0 \n 1 \n {} {} {}\n".format(inputDimensions,
outputDimensions, nrows))
output.write(final_file)
def execute(self, context):
self.report({'INFO'}, "Animating armature %s..." %
context.scene.speech2anim_data.training_videos_path)
os.chdir(
bpy.utils.user_resource("SCRIPTS", "addons") + config.ADDON_PATH +
'/src')
reloadAnimationConfig(context)
d = context.scene.speech2anim_data
result_path = DataManager.Predict(d.animate_model_path, d.input_file)
data = CSVFile()
data.from_file(result_path)
data.tidy()
data = data.to_dict()
data = BlenderManager.fillFrameIndexes(data)
armature = context.object
BlenderManager.plotData(armature, data)
BlenderManager.animate(armature, data)
#load audio
context.scene.sequence_editor_clear()
#TODO:refactor, promote to function
for area in bpy.context.screen.areas:
if area.type == 'SEQUENCE_EDITOR':
space_data = area.spaces.active
override = {
'area': area,
'screen': context.screen,
'region': context.region,
'window': context.window,
'scene': context.scene,
'blend_data': context.blend_data
}
bpy.ops.sequencer.sound_strip_add(override,
filepath=d.input_file,
frame_start=0)
return {'FINISHED'}
def invoke(self, context, event):
scn = context.scene
idx = scn.speech2anim_data.selected_training_video_index
addon_path = wops.clear(
bpy.utils.user_resource("SCRIPTS", "addons") + config.ADDON_PATH)
try:
item = scn.speech2anim_data.training_videos_list[idx]
except IndexError:
pass
else:
if self.action == 'CLEAR_POSE':
filename = item.name + '.avi'
BlenderManager.erase_sequence(filename)
wops.delete(addon_path + '/output/openpose/renders/' +
filename)
wops.rmdir('output/openpose/' + item.name, addon_path)
self.report({'INFO'},
'Pose cleared for video {}'.format(item.name))
elif self.action == 'CLEAR_AUDIO':
filename = item.name + '.wav'
#BlenderManager.erase_sequence(filename)
#wops.delete(addon_path+'/tempAudioFiles/'+filename)
wops.rmdir('output/opensmile/' + item.name, addon_path)
self.report({'INFO'},
'Audio cleared for video {}'.format(item.name))
elif self.action == 'SEE_INFO':
if not context.object.type == 'ARMATURE':
self.report({'ERROR'}, 'Please, select an armature')
return {"CANCELLED"}
data = CSVFile()
data.from_file(addon_path + '/output/tempdata/' + item.name +
'/training_info.csv')
data = data.to_dict()
data_funcs = CSVFile()
data_funcs.from_file(addon_path + '/output/tempdata/' +
item.name + '/training_info_funcs.csv')
data_funcs = data_funcs.to_dict()
all_data = {**data_funcs, **data}
BlenderManager.plotData(context.object, all_data)
#load video and audio
context.scene.sequence_editor_clear()
space_data = None
#TODO:refactor, promote to function
for area in bpy.context.screen.areas:
if area.type == 'SEQUENCE_EDITOR':
space_data = area.spaces.active
override = {
'area': area,
'screen': context.screen,
'region': context.region,
'window': context.window,
'scene': context.scene,
'blend_data': context.blend_data
}
if os.path.isfile(addon_path +
'/output/openpose/renders/' +
item.name + '.avi'):
bpy.ops.sequencer.movie_strip_add(
override,
filepath=addon_path +
'/output/openpose/renders/' + item.name +
'.avi',
frame_start=0)
else:
print(
"Can't find pose video:",
addon_path + '/output/openpose/renders/' +
item.name + '.avi')
if os.path.isfile(addon_path + '/tempAudioFiles/' +
item.name + '.wav'):
bpy.ops.sequencer.sound_strip_add(
override,
filepath=addon_path + '/tempAudioFiles/' +
item.name + '.wav',
frame_start=0)
else:
print(
"Can't find audio file:", addon_path +
'/tempAudioFiles/' + item.name + '.wav')
self.report({'INFO'},
'Seeing info for video {}'.format(item.name))
return {"FINISHED"}
def GenerateIndividualTrainingData(
training_videos_folder=config.TRAINING_VIDEOS_FOLDER):
for filename in glob.glob(training_videos_folder + '/*'):
video_path = w2p(filename)
if not os.path.isfile(w2p(filename)):
print("Skipping folder " + filename + "...")
continue
video_name = (video_path.split('/')[-1]).split('.')[0]
output_folder = config.TEMPDATA_OUTPUT + '/' + video_name
if len(glob.glob(output_folder + '/*')) > 0:
print("Skipping training for " + video_name + "...")
continue
print("Generating training data for " + video_name + "...")
mkdir(output_folder)
#array of poses
keypoints = training.getKeypoints(config.OPENPOSE_OUTPUT + '/' +
video_name)
#get current video framerate
framerate = utils.get_frame_rate(video_path)
#array of window values (dicts valueName:value)
computedWindows = training.computeWindows(keypoints,
config.WINDOW_VALUES,
framerate,
config.POSE_WINDOW_SIZE_MS,
config.POSE_WINDOW_STEP_MS)
#dict with statistics about window values (valueName_func:value)
computedFuncs = training.computeFunctionals(computedWindows,
config.FUNCTIONALS)
cfuncs_names = list(computedFuncs.keys())
func_outputfile = CSVFile(['frameIndex'] + cfuncs_names)
#frame index 0
row = [0]
for name in cfuncs_names:
row.append(computedFuncs[name])
func_outputfile.addRow(row)
func_outputfile.saveAs(output_folder + '/training_info_funcs')
#array of arrays of labels (labels per window)
#labels = {x | |x| = len(config.LABEL_GROUPS) ^ x_i = label of group i}
labels = training.labelWindows(computedWindows, computedFuncs,
config.LABEL_GROUPS)
#create info output file
window_values_names = list(computedWindows[0].keys())
#array of label names by order of definition
label_names = utils.get_label_names(config.LABEL_GROUPS)
labels_outputfile = CSVFile(['frameIndex'] + window_values_names +
label_names)
#step in frame count (frames/window)
frames_per_pose_window = (config.POSE_WINDOW_STEP_MS /
1000) * framerate
#for each window
for i, w in enumerate(computedWindows):
#frame index
row = [round(i * frames_per_pose_window)]
#add all computed values (from pose data)
for name in window_values_names:
row.append(w[name])
#window_labels contains a label for each group
window_labels = labels[i]
#for every group
for j, g in enumerate(config.LABEL_GROUPS):
#get the label of that group
label_of_g = window_labels[j]
#0 is null class
for k in range(1, len(g['label_names']) + 1):
row.append(int(k == label_of_g))
labels_outputfile.addRow(row)
labels_outputfile.saveAs(output_folder + '/training_info')
#create training data
audio_features = CSVFile()
audio_features.from_file(config.OPENSMILE_OUTPUT + '/' + video_name +
'/features.csv')
label_col = []
#output file (without frametime col)
regression_training_data = CSVFile(audio_features.getHeader()[1:] +
window_values_names)
group_names = utils.get_group_names(config.LABEL_GROUPS)
training_data = CSVFile(audio_features.getHeader()[1:] + group_names)
for row in audio_features.getRows():
#row[0] is the tick number
#frame in which the tick is contained
frame = math.floor(
((int(row[0]) * config.AUDIO_WINDOW_SIZE_MS) / 1000) *
framerate)
#window in which the frame is contained
window = min(math.floor(frame / frames_per_pose_window),
len(computedWindows) - 1)
#add classification row
training_data.addRow(row[1:] + labels[window])
#create regression row (append pose features to audio features)
row_regression = row[1:]
for name in window_values_names:
row_regression.append(computedWindows[window][name])
regression_training_data.addRow(row_regression)
training_data.saveAs(output_folder + '/training_data')
regression_training_data.saveAs(output_folder +
'/regression_training_data')