def gen_multi_seqs():
global start_time
start_time = time.time()
log_message("Remove out/, and tmp/")
if os.path.exists("../out/"):
shutil.rmtree("../out/")
if os.path.exists("../tmp/"):
shutil.rmtree("../tmp/")
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
print("Resource information: ")
bg_splits, cloth_splits, nseqs, nshapes = print_resource_info(
bg_path, smpl_data_folder, smpl_data_filename)
for gender in ["female", "male"]:
for split in ["train", "test"]:
for irun in range(100):
nbg = len(bg_splits[split])
ncloth = len(cloth_splits[gender + "_" + split])
# setup params
setting = edict()
setting.split = split ##
setting.gender = gender ## gender, 0/1
setting.iseq = random.randint(0, nseqs) ## seq id, rand
setting.ibg = random.randint(0, nbg) ## bg image, rand
setting.ishape = random.randint(0, nshapes) ## shape idx, rand
setting.icloth = random.randint(0, ncloth) ## cloth idx, rand
setting.irun = irun
print("Generate a sequence with settings: ", setting)
nframes = gen_one_seqs(setting)
print("#%d frames are generated." % (nframes))
def main():
# time logging
global start_time
start_time = time.time()
import argparse
# parse commandline arguments
log_message(sys.argv)
parser = argparse.ArgumentParser(description='Generate synth dataset images.')
parser.add_argument('--idx', type=int,
help='idx of the requested sequence')
parser.add_argument('--ishape', type=int,
help='requested cut, according to the stride')
parser.add_argument('--stride', type=int,
help='stride amount, default 50')
args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:])
idx = args.idx
ishape = args.ishape
stride = args.stride
log_message("input idx: %d" % idx)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
if idx == None:
exit(1)
if ishape == None:
exit(1)
if stride == None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# import idx info (name, split)
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
idx_info = [x for x in idx_info if x['name'][:4] != 'h36m']
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
resy = params['resy']
resx = params['resx']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
# compute number of cuts
nb_ishape = max(1, int(np.ceil((idx_info['nb_frames'] - (clipsize - stride))/stride)))
log_message("Max ishape: %d" % (nb_ishape - 1))
if ishape == None:
exit(1)
assert(ishape < nb_ishape)
# name is set given idx
name = idx_info['name']
output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
params['output_path'] = output_path
tmp_path = join(tmp_path, 'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
params['tmp_path'] = tmp_path
# check if already computed
# + clean up existing tmp folders if any
if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
os.system('rm -rf %s' % tmp_path)
rgb_vid_filename = "%s_c%04d.mp4" % (join(output_path, name.replace(' ', '')), (ishape + 1))
#if os.path.isfile(rgb_vid_filename):
# log_message("ALREADY COMPUTED - existing: %s" % rgb_vid_filename)
# return 0
# create tmp directory
if not exists(tmp_path):
mkdir_safe(tmp_path)
# >> don't use random generator before this point <<
# initialize RNG with seeds from sequence id
import hashlib
s = "synth_data:%d:%d:%d" % (idx, runpass,ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10 ** 8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
if(output_types['vblur']):
vblur_factor = np.random.normal(0.5, 0.5)
params['vblur_factor'] = vblur_factor
log_message("Setup Blender")
# create copy-spher.harm. directory if not exists
sh_dir = join(tmp_path, 'spher_harm')
if not exists(sh_dir):
mkdir_safe(sh_dir)
sh_dst = join(sh_dir, 'sh_%02d_%05d.osl' % (runpass, idx))
os.system('cp spher_harm/sh.osl %s' % sh_dst)
genders = {0: 'female', 1: 'male'}
# pick random gender
gender = choice(genders)
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
log_message("Listing background images")
bg_names = join(bg_path, '%s_img.txt' % idx_info['use_split'])
nh_txt_paths = []
with open(bg_names) as f:
for line in f:
nh_txt_paths.append(join(bg_path, line))
# grab clothing names
log_message("clothing: %s" % clothing_option)
with open( join(smpl_data_folder, 'textures', '%s_%s.txt' % ( gender, idx_info['use_split'] ) ) ) as f:
txt_paths = f.read().splitlines()
# if using only one source of clothing
if clothing_option == 'nongrey':
txt_paths = [k for k in txt_paths if 'nongrey' in k]
elif clothing_option == 'grey':
txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
cloth_img_name = choice(txt_paths)
cloth_img_name = join(smpl_data_folder, cloth_img_name)
cloth_img = bpy.data.images.load(cloth_img_name)
# random background
bg_img_name = choice(nh_txt_paths)[:-1]
bg_img = bpy.data.images.load(bg_img_name)
log_message("Loading parts segmentation")
beta_stds = np.load(join(smpl_data_folder, ('%s_beta_stds.npy' % gender)))
log_message("Building materials tree")
mat_tree = bpy.data.materials['Material'].node_tree
create_sh_material(mat_tree, sh_dst, cloth_img)
res_paths = create_composite_nodes(scene.node_tree, params, img=bg_img, idx=idx)
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
camera_distance = np.random.normal(8.0, 1)
params['camera_distance'] = camera_distance
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = True #True: 0-24, False: expected to have 0-1 bg/fg
log_message("Creating materials segmentation")
# create material segmentation
if segmented_materials:
materials = create_segmentation(ob, params)
prob_dressed = {'leftLeg':.5, 'leftArm':.9, 'leftHandIndex1':.01,
'rightShoulder':.8, 'rightHand':.01, 'neck':.01,
'rightToeBase':.9, 'leftShoulder':.8, 'leftToeBase':.9,
'rightForeArm':.5, 'leftHand':.01, 'spine':.9,
'leftFoot':.9, 'leftUpLeg':.9, 'rightUpLeg':.9,
'rightFoot':.9, 'head':.01, 'leftForeArm':.5,
'rightArm':.5, 'spine1':.9, 'hips':.9,
'rightHandIndex1':.01, 'spine2':.9, 'rightLeg':.5}
else:
materials = {'FullBody': bpy.data.materials['Material']}
prob_dressed = {'FullBody': .6}
orig_pelvis_loc = (arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy()) - Vector((-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data, ob, obname, idx=idx, gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
if idx_info['use_split'] == 'train':
fshapes = fshapes[:int(nb_fshapes*0.8)]
elif idx_info['use_split'] == 'test':
fshapes = fshapes[int(nb_fshapes*0.8):]
# pick random real body shape
shape = choice(fshapes) #+random_shape(.5) can add noise
#shape = random_shape(3.) # random body shape
# example shapes
#shape = np.zeros(10) #average
#shape = np.array([ 2.25176191, -3.7883464 , 0.46747496, 3.89178988, 2.20098416, 0.26102114, -3.07428093, 0.55708514, -3.94442258, -2.88552087]) #fat
#shape = np.array([-2.26781107, 0.88158132, -0.93788176, -0.23480508, 1.17088298, 1.55550789, 0.44383225, 0.37688275, -0.27983086, 1.77102953]) #thin
#shape = np.array([ 0.00404852, 0.8084637 , 0.32332591, -1.33163664, 1.05008727, 1.60955275, 0.22372946, -0.10738459, 0.89456312, -1.22231216]) #short
#shape = np.array([ 3.63453289, 1.20836171, 3.15674431, -0.78646793, -1.93847355, -0.32129994, -0.97771656, 0.94531640, 0.52825811, -0.99324327]) #tall
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname+'_Pelvis'].location).copy()
# create output directory
if not exists(output_path):
mkdir_safe(output_path)
# spherical harmonics material needs a script to be loaded and compiled
scs = []
for mname, material in materials.items():
scs.append(material.node_tree.nodes['Script'])
scs[-1].filepath = sh_dst
scs[-1].update()
rgb_dirname = name.replace(" ", "") + '_c%04d.mp4' % (ishape + 1)
rgb_path = join(tmp_path, rgb_dirname)
data = cmu_parms[name]
fbegin = ishape*stepsize*stride
fend = min(ishape*stepsize*stride + stepsize*clipsize, len(data['poses']))
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
nframes = len(data['poses'][::stepsize])
matfile_info = join(output_path, name.replace(" ", "") + "_c%04d_info.mat" % (ishape+1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
dict_info['bg'] = np.zeros((N,), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty(3) # (1, 3)
dict_info['clipNo'] = ishape +1
dict_info['cloth'] = np.zeros((N,), dtype=np.object) # clothing texture image path
dict_info['gender'] = np.empty(N, dtype='uint8') # 0 for male, 1 for female
dict_info['joints2D'] = np.empty((2, 24, N), dtype='float32') # 2D joint positions in pixel space
dict_info['joints3D'] = np.empty((3, 24, N), dtype='float32') # 3D joint positions in world coordinates
dict_info['light'] = np.empty((9, N), dtype='float32')
dict_info['pose'] = np.empty((data['poses'][0].size, N), dtype='float32') # joint angles from SMPL (CMU)
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
if(output_types['vblur']):
dict_info['vblur_factor'] = np.empty(N, dtype='float32')
# for each clipsize'th frame in the sequence
get_real_frame = lambda ifr: ifr
random_zrot = 0
reset_loc = False
batch_it = 0
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob, obname, scene,
cam_ob, smpl_data['regression_verts'], smpl_data['joint_regressor'])
random_zrot = 2*np.pi*np.random.rand()
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
arm_ob.rotation_euler.x -= math.pi / 2
# create a keyframe animation with pose, translation, blendshapes and camera motion
# LOOP TO CREATE 3D ANIMATION
for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob, obname, scene, cam_ob, get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(genders.values()).index(gender)]
if(output_types['vblur']):
dict_info['vblur_factor'][iframe] = vblur_factor
arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion(Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname+'_root'].keyframe_insert('rotation_quaternion', frame=get_real_frame(seq_frame))
dict_info['zrot'][iframe] = random_zrot
scene.update()
# Bodies centered only in each minibatch of clipsize frames
if seq_frame == 0 or reset_loc:
reset_loc = False
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy()
rotated_orig = Vector([orig_pelvis_loc.copy()[0], orig_pelvis_loc.copy()[2], -orig_pelvis_loc.copy()[1]])
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() - rotated_orig.copy())
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
dict_info['camLoc'] = np.array(cam_ob.location)
scene.node_tree.nodes['Image'].image = bg_img
for part, material in materials.items():
material.node_tree.nodes['Vector Math'].inputs[1].default_value[:2] = (0, 0)
# random light
sh_coeffs = .7 * (2 * np.random.rand(9) - 1)
sh_coeffs[0] = .5 + .9 * np.random.rand() # Ambient light (first coeff) needs a minimum is ambient. Rest is uniformly distributed, higher means brighter.
sh_coeffs[1] = -.7 * np.random.rand()
for ish, coeff in enumerate(sh_coeffs):
for sc in scs:
sc.inputs[ish+1].default_value = coeff
# iterate over the keyframes and render
# LOOP TO RENDER
for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
scene.frame_set(get_real_frame(seq_frame))
iframe = seq_frame
dict_info['bg'][iframe] = bg_img_name
dict_info['cloth'][iframe] = cloth_img_name
dict_info['light'][:, iframe] = sh_coeffs
scene.render.use_antialiasing = False
scene.render.filepath = join(rgb_path, 'Image%04d.png' % get_real_frame(seq_frame))
log_message("Rendering frame %d" % seq_frame)
# disable render output
logfile = '/dev/null'
open(logfile, 'a').close()
old = os.dup(1)
sys.stdout.flush()
os.close(1)
os.open(logfile, os.O_WRONLY)
# Render
bpy.ops.render.render(write_still=True)
# disable output redirection
os.close(1)
os.dup(old)
os.close(old)
# NOTE:
# ideally, pixels should be readable from a viewer node, but I get only zeros
# --> https://ammous88.wordpress.com/2015/01/16/blender-access-render-results-pixels-directly-from-python-2/
# len(np.asarray(bpy.data.images['Render Result'].pixels) is 0
# Therefore we write them to temporary files and read with OpenEXR library (available for python2) in main_part2.py
# Alternatively, if you don't want to use OpenEXR library, the following commented code does loading with Blender functions, but it can cause memory leak.
# If you want to use it, copy necessary lines from main_part2.py such as definitions of dict_normal, matfile_normal...
#for k, folder in res_paths.items():
# if not k== 'vblur' and not k=='fg':
# path = join(folder, 'Image%04d.exr' % get_real_frame(seq_frame))
# render_img = bpy.data.images.load(path)
# # render_img.pixels size is width * height * 4 (rgba)
# arr = np.array(render_img.pixels[:]).reshape(resx, resy, 4)[::-1,:, :] # images are vertically flipped
# if k == 'normal':# 3 channels, original order
# mat = arr[:,:, :3]
# dict_normal['normal_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'gtflow':
# mat = arr[:,:, 1:3]
# dict_gtflow['gtflow_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'depth':
# mat = arr[:,:, 0]
# dict_depth['depth_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'segm':
# mat = arr[:,:,0]
# dict_segm['segm_%d' % (iframe + 1)] = mat.astype(np.uint8, copy=False)
#
# # remove the image to release memory, object handles, etc.
# render_img.user_clear()
# bpy.data.images.remove(render_img)
# bone locations should be saved after rendering so that the bones are updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene, cam_ob)
dict_info['joints2D'][:, :, iframe] = np.transpose(bone_locs_2D)
dict_info['joints3D'][:, :, iframe] = np.transpose(bone_locs_3D)
reset_loc = (bone_locs_2D.max(axis=-1) > 256).any() or (bone_locs_2D.min(axis=0) < 0).any()
arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion((1, 0, 0, 0))
# save a .blend file for debugging:
# bpy.ops.wm.save_as_mainfile(filepath=join(tmp_path, 'pre.blend'))
# save RGB data with ffmpeg (if you don't have h264 codec, you can replace with another one and control the quality with something like -q:v 3)
cmd_ffmpeg = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 ''%s_c%04d.mp4''' % (join(rgb_path, 'Image%04d.png'), join(output_path, name.replace(' ', '')), (ishape + 1))
log_message("Generating RGB video (%s)" % cmd_ffmpeg)
os.system(cmd_ffmpeg)
if(output_types['vblur']):
cmd_ffmpeg_vblur = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 -vf "scale=trunc(iw/2)*2:trunc(ih/2)*2" ''%s_c%04d.mp4''' % (join(res_paths['vblur'], 'Image%04d.png'), join(output_path, name.replace(' ', '')+'_vblur'), (ishape + 1))
log_message("Generating vblur video (%s)" % cmd_ffmpeg_vblur)
os.system(cmd_ffmpeg_vblur)
if(output_types['fg']):
cmd_ffmpeg_fg = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 ''%s_c%04d.mp4''' % (join(res_paths['fg'], 'Image%04d.png'), join(output_path, name.replace(' ', '')+'_fg'), (ishape + 1))
log_message("Generating fg video (%s)" % cmd_ffmpeg_fg)
os.system(cmd_ffmpeg_fg)
cmd_tar = 'tar -czvf %s/%s.tar.gz -C %s %s' % (output_path, rgb_dirname, tmp_path, rgb_dirname)
log_message("Tarballing the images (%s)" % cmd_tar)
os.system(cmd_tar)
# save annotation excluding png/exr data to _info.mat file
import scipy.io
scipy.io.savemat(matfile_info, dict_info, do_compression=True)
def main():
# time logging
global start_time
start_time = time.time()
import argparse
# parse commandline arguments
log_message(sys.argv)
parser = argparse.ArgumentParser(
description='Generate synth dataset images.')
parser.add_argument('--idx',
type=int,
help='idx of the requested sequence')
parser.add_argument('--name',
type=str,
help='name of the requested sequence')
parser.add_argument('--ishape',
type=int,
help='requested cut, according to the stride')
parser.add_argument('--stride', type=int, help='stride amount, default 50')
parser.add_argument('--direction',
type=str,
help='subject direction, default forward')
parser.add_argument('--subject_id',
type=int,
help='local subject id, default 0')
args = parser.parse_args(sys.argv[sys.argv.index("---") + 1:])
idx = args.idx
name = args.name
ishape = args.ishape
stride = args.stride
direction = args.direction
subject_id = args.subject_id
log_message("input idx: %d" % idx)
log_message("input name: %s" % name)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
log_message("Subject direction: %s" % direction)
log_message("Local subject id: %d" % subject_id)
if idx == None:
exit(1)
if ishape == None:
exit(1)
if stride == None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# import idx info (name, split)
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
for dic in idx_info:
if dic['name'] == name:
idx_info = dic
break
else:
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
#log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
resy = params['resy']
resx = params['resx']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
# compute number of cuts
nb_ishape = max(
1, int(np.ceil(
(idx_info['nb_frames'] - (clipsize - stride)) / stride)))
log_message("Max ishape: %d" % (nb_ishape - 1))
if ishape == None:
exit(1)
assert (ishape < nb_ishape)
# name is set given idx
name = idx_info['name']
output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
params['output_path'] = output_path
tmp_path = join(
tmp_path,
'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
params['tmp_path'] = tmp_path
# check if already computed
# + clean up existing tmp folders if any
if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
os.system('rm -rf %s' % tmp_path)
rgb_vid_filename = "%s_c%04d.mp4" % (join(output_path, name.replace(
' ', '')), (ishape + 1))
# create tmp directory
if not exists(tmp_path):
mkdir_safe(tmp_path)
# >> don't use random generator before this point <<
# initialize RNG with seeds from sequence id
import hashlib
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10**
8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
if (output_types['vblur']):
vblur_factor = np.random.normal(0.5, 0.5)
params['vblur_factor'] = vblur_factor
log_message("Setup Blender")
# create copy-spher.harm. directory if not exists
sh_dir = join(tmp_path, 'spher_harm')
if not exists(sh_dir):
mkdir_safe(sh_dir)
sh_dst = join(sh_dir, 'sh_%02d_%05d.osl' % (runpass, idx))
os.system('cp spher_harm/sh.osl %s' % sh_dst)
genders = {0: 'male', 1: 'female'}
# pick random gender
gender = genders[sum(divmod(subject_id, 2)) %
2] #genders[subject_id % 2]#choice(genders)
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
log_message("Listing background images")
#bg_names = join(bg_path, '%s_img.txt' % idx_info['use_split'])
bg_names = join(bg_path, 'bg.txt')
nh_txt_paths = []
with open(bg_names) as f:
for line in f:
nh_txt_paths.append(join(bg_path, line))
# grab clothing names
log_message("clothing: %s" % clothing_option)
with open(join(smpl_data_folder, 'textures',
'%s_train.txt' % gender)) as f:
txt_paths = f.read().splitlines()
# if using only one source of clothing
if clothing_option == 'nongrey':
txt_paths = [k for k in txt_paths if 'nongrey' in k]
elif clothing_option == 'grey':
txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
cloth_img_name = choice(
txt_paths) #txt_paths[subject_id]#choice(txt_paths)
cloth_img_name = join(smpl_data_folder, cloth_img_name)
cloth_img = bpy.data.images.load(cloth_img_name)
# random background
bg_img_name = choice(nh_txt_paths)[:-1]
bg_img = bpy.data.images.load(bg_img_name)
log_message("Loading parts segmentation")
beta_stds = np.load(join(smpl_data_folder, ('%s_beta_stds.npy' % gender)))
log_message("Building materials tree")
mat_tree = bpy.data.materials['Material'].node_tree
create_sh_material(mat_tree, sh_dst, cloth_img)
res_paths = create_composite_nodes(scene.node_tree,
params,
img=bg_img,
idx=idx)
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
camera_distance = 11.0 #np.random.normal(8.0, 1)
params['camera_distance'] = camera_distance
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = True #True: 0-24, False: expected to have 0-1 bg/fg
log_message("Creating materials segmentation")
# create material segmentation
if segmented_materials:
materials = create_segmentation(ob, params)
prob_dressed = {
'leftLeg': .5,
'leftArm': .9,
'leftHandIndex1': .01,
'rightShoulder': .8,
'rightHand': .01,
'neck': .01,
'rightToeBase': .9,
'leftShoulder': .8,
'leftToeBase': .9,
'rightForeArm': .5,
'leftHand': .01,
'spine': .9,
'leftFoot': .9,
'leftUpLeg': .9,
'rightUpLeg': .9,
'rightFoot': .9,
'head': .01,
'leftForeArm': .5,
'rightArm': .5,
'spine1': .9,
'hips': .9,
'rightHandIndex1': .01,
'spine2': .9,
'rightLeg': .5
}
else:
materials = {'FullBody': bpy.data.materials['Material']}
prob_dressed = {'FullBody': .6}
orig_pelvis_loc = None
random_zrot = get_zrot(name, direction)
if direction == 'forward':
orig_pelvis_loc = (
arm_ob.matrix_world.copy() *
arm_ob.pose.bones[obname + '_Pelvis'].head.copy()) - Vector(
(-1., 0.75, -1.3))
elif direction == 'backward':
orig_pelvis_loc = (
arm_ob.matrix_world.copy() *
arm_ob.pose.bones[obname + '_Pelvis'].head.copy()) - Vector(
(-1., 0.75, 3.1))
orig_cam_loc = cam_ob.location.copy()
print("CAM LOC:", orig_cam_loc, type(orig_cam_loc))
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data,
ob,
obname,
name,
gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
#if idx_info['use_split'] == 'train':
# fshapes = fshapes[:int(nb_fshapes*0.8)]
#elif idx_info['use_split'] == 'test':
# fshapes = fshapes[int(nb_fshapes*0.8):]
# pick random real body shape
shape = fshapes[
subject_id %
nb_fshapes] #+random_shape(.5)#choice(fshapes) #+random_shape(.5) can add noise
#shape = random_shape(3.) # random body shape
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname +
'_Pelvis'].location).copy()
# create output directory
if not exists(output_path):
mkdir_safe(output_path)
# spherical harmonics material needs a script to be loaded and compiled
scs = []
for mname, material in materials.items():
scs.append(material.node_tree.nodes['Script'])
scs[-1].filepath = sh_dst
scs[-1].update()
rgb_dirname = name.replace(" ", "") + '_c%04d.mp4' % (ishape + 1)
rgb_path = join(tmp_path, rgb_dirname)
data = cmu_parms[name]
data = cut_sequence(name, data)
fbegin = ishape * stepsize * stride
fend = min(ishape * stepsize * stride + stepsize * clipsize,
len(data['poses']))
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
nframes = len(data['poses'][::stepsize])
matfile_info = join(
output_path,
name.replace(" ", "") + "_c%04d_info.mat" % (ishape + 1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
dict_info['bg'] = np.zeros((N, ), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty(3) # (1, 3)
dict_info['clipNo'] = ishape + 1
dict_info['cloth'] = np.zeros(
(N, ), dtype=np.object) # clothing texture image path
dict_info['gender'] = np.empty(N,
dtype='uint8') # 0 for male, 1 for female
dict_info['joints2D'] = np.empty(
(2, 24, N), dtype='float32') # 2D joint positions in pixel space
dict_info['joints3D'] = np.empty(
(3, 24, N), dtype='float32') # 3D joint positions in world coordinates
dict_info['light'] = np.empty((9, N), dtype='float32')
dict_info['pose'] = np.empty(
(data['poses'][0].size, N),
dtype='float32') # joint angles from SMPL (CMU)
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
if (output_types['vblur']):
dict_info['vblur_factor'] = np.empty(N, dtype='float32')
# for each clipsize'th frame in the sequence
get_real_frame = lambda ifr: ifr
reset_loc = False
batch_it = 0
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob, obname,
scene, cam_ob,
smpl_data['regression_verts'],
smpl_data['joint_regressor'])
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
# create a keyframe animation with pose, translation, blendshapes and camera motion
# LOOP TO CREATE 3D ANIMATION
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# Change shape
if is_arbitrary_shape and iframe % 2 == 0:
shape = choice(fshapes)
shape += np.random.normal(0, .1, shape.shape)
# apply the translation, pose and shape to the character
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob, obname,
scene, cam_ob, get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(
genders.values()).index(gender)]
if (output_types['vblur']):
dict_info['vblur_factor'][iframe] = vblur_factor
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
dict_info['zrot'][iframe] = random_zrot
scene.update()
# Bodies centered only in each minibatch of clipsize frames
if seq_frame == 0 or reset_loc:
reset_loc = False
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() -
orig_pelvis_loc.copy())
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
dict_info['camLoc'] = np.array(cam_ob.location)
scene.node_tree.nodes['Image'].image = bg_img
for part, material in materials.items():
material.node_tree.nodes['Vector Math'].inputs[1].default_value[:2] = (
0, 0)
# random light
sh_coeffs = .7 * (2 * np.random.rand(9) - 1)
sh_coeffs[0] = .5 + .9 * np.random.rand(
) # Ambient light (first coeff) needs a minimum is ambient. Rest is uniformly distributed, higher means brighter.
sh_coeffs[1] = -.7 * np.random.rand()
for ish, coeff in enumerate(sh_coeffs):
for sc in scs:
sc.inputs[ish + 1].default_value = coeff
# iterate over the keyframes and render
# LOOP TO RENDER
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
scene.frame_set(get_real_frame(seq_frame))
iframe = seq_frame
dict_info['bg'][iframe] = bg_img_name
dict_info['cloth'][iframe] = cloth_img_name
dict_info['light'][:, iframe] = sh_coeffs
img_path = join(rgb_path, 'Image%04d.png' % get_real_frame(seq_frame))
scene.render.use_antialiasing = False
scene.render.filepath = img_path
log_message("Rendering frame %d" % seq_frame)
# disable render output
logfile = '/dev/null'
open(logfile, 'a').close()
old = os.dup(1)
sys.stdout.flush()
os.close(1)
os.open(logfile, os.O_WRONLY)
# Render
bpy.ops.render.render(write_still=True)
# disable output redirection
os.close(1)
os.dup(old)
os.close(old)
# bone locations should be saved after rendering so that the bones are updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene,
cam_ob)
dict_info['joints2D'][:, :, iframe] = np.transpose(bone_locs_2D)
dict_info['joints3D'][:, :, iframe] = np.transpose(bone_locs_3D)
#Draw skeleton
if is_visualization:
draw_skeleton(img_path, dict_info['joints2D'][:, :, iframe])
reset_loc = (bone_locs_2D.max(axis=-1) >
256).any() or (bone_locs_2D.min(axis=0) < 0).any()
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
(1, 0, 0, 0))
# save a .blend file for debugging:
# bpy.ops.wm.save_as_mainfile(filepath=join(tmp_path, 'pre.blend'))
# save RGB data with ffmpeg (if you don't have h264 codec, you can replace with another one and control the quality with something like -q:v 3)
cmd_ffmpeg = 'ffmpeg -y -r 25 -i ' '%s' ' -c:v h264 -pix_fmt yuv420p -crf 23 ' '%s_c%04d.mp4' '' % (
join(rgb_path, 'Image%04d.png'),
join(output_path, name.replace(' ', '')), (ishape + 1))
log_message("Generating RGB video (%s)" % cmd_ffmpeg)
os.system(cmd_ffmpeg)
if (output_types['vblur']):
cmd_ffmpeg_vblur = 'ffmpeg -y -r 25 -i ' '%s' ' -c:v h264 -pix_fmt yuv420p -crf 23 -vf "scale=trunc(iw/2)*2:trunc(ih/2)*2" ' '%s_c%04d.mp4' '' % (
join(res_paths['vblur'], 'Image%04d.png'),
join(output_path,
name.replace(' ', '') + '_vblur'), (ishape + 1))
log_message("Generating vblur video (%s)" % cmd_ffmpeg_vblur)
os.system(cmd_ffmpeg_vblur)
if (output_types['fg']):
cmd_ffmpeg_fg = 'ffmpeg -y -r 25 -i ' '%s' ' -c:v h264 -pix_fmt yuv420p -crf 23 ' '%s_c%04d.mp4' '' % (
join(res_paths['fg'], 'Image%04d.png'),
join(output_path,
name.replace(' ', '') + '_fg'), (ishape + 1))
log_message("Generating fg video (%s)" % cmd_ffmpeg_fg)
os.system(cmd_ffmpeg_fg)
cmd_tar = 'tar -czvf %s/%s.tar.gz -C %s %s' % (output_path, rgb_dirname,
tmp_path, rgb_dirname)
log_message("Tarballing the images (%s)" % cmd_tar)
os.system(cmd_tar)
# save annotation excluding png/exr data to _info.mat file
import scipy.io
scipy.io.savemat(matfile_info, dict_info, do_compression=True)
log_message("start part 2")
import hashlib
import random
# initialize random seeds with sequence id
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10**
8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
# import configuration
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
resy = params['resy']
resx = params['resx']
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
# check whether openexr_py2_path is loaded from configuration file
if 'openexr_py2_path' in locals() or 'openexr_py2_path' in globals():
for exr_path in openexr_py2_path.split(':'):
def main():
# time logging
global start_time
start_time = time.time()
import argparse
# parse commandline arguments
#log_message(sys.argv)
parser = argparse.ArgumentParser(
description='Generate synth dataset images.')
parser.add_argument('--idx',
type=int,
help='idx of the requested sequence')
parser.add_argument('--ishape',
type=int,
help='requested cut, according to the stride')
parser.add_argument('--stride', type=int, help='stride amount, default 50')
parser.add_argument('--gender', type=str, help='gender {male, female}')
parser.add_argument(
'--body_shape_idx',
type=int,
help=
'body shape idx (height, weight etc.) < 1682 for female, < 1360 for male)'
)
parser.add_argument('--outdir', type=str, help='out directory')
args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:])
idx = args.idx
ishape = args.ishape
stride = args.stride
gender = args.gender
body_shape_idx = args.body_shape_idx
outdir = args.outdir
log_message("input idx: %d" % idx)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
log_message("gender: %s" % gender)
log_message("body_shape_idx: %d" % body_shape_idx)
if idx == None:
exit(1)
if ishape == None:
exit(1)
if stride == None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# Custom checks for specially added fields (gender & body_shape_idx)
if gender is 'male':
assert body_shape_idx < 1360
elif gender is 'female':
assert body_shape_idx < 1682
else:
assert (gender in ['male', 'female'])
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
# compute number of cuts
nb_ishape = max(
1, int(np.ceil(
(idx_info['nb_frames'] - (clipsize - stride)) / stride)))
log_message("Max ishape: %d" % (nb_ishape - 1))
if ishape == None:
exit(1)
assert (ishape < nb_ishape)
# name is set given idx
name = idx_info['name']
output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
params['output_path'] = output_path
tmp_path = join(
tmp_path,
'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
params['tmp_path'] = tmp_path
# check if already computed
# + clean up existing tmp folders if any
if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
os.system('rm -rf %s' % tmp_path)
# >> don't use random generator before this point <<
# initialize RNG with seeds from sequence id
import hashlib
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10**
8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
if (output_types['vblur']):
vblur_factor = np.random.normal(0.5, 0.5)
params['vblur_factor'] = vblur_factor
log_message("Setup Blender")
genders = {0: 'female', 1: 'male'}
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
# grab clothing names
log_message("clothing: %s" % clothing_option)
with open(
join(smpl_data_folder, 'textures',
'%s_%s.txt' % (gender, idx_info['use_split']))) as f:
txt_paths = f.read().splitlines()
# if using only one source of clothing
if clothing_option == 'nongrey':
txt_paths = [k for k in txt_paths if 'nongrey' in k]
elif clothing_option == 'grey':
txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
cloth_img_name = choice(txt_paths)
cloth_img_name = join(smpl_data_folder, cloth_img_name)
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
camera_distance = 4.0 # Not rendering images so camera distance can be any #
params['camera_distance'] = camera_distance
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
orig_pelvis_loc = (
arm_ob.matrix_world.copy() *
arm_ob.pose.bones[obname + '_Pelvis'].head.copy()) - Vector(
(-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data,
ob,
obname,
idx=idx,
gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
# Force the train split
fshapes = fshapes[:int(nb_fshapes * 0.8)]
shape = fshapes[body_shape_idx]
# example shapes
#shape = np.zeros(10) #average
#shape = np.array([ 2.25176191, -3.7883464 , 0.46747496, 3.89178988, 2.20098416, 0.26102114, -3.07428093, 0.55708514, -3.94442258, -2.88552087]) #fat
#shape = np.array([-2.26781107, 0.88158132, -0.93788176, -0.23480508, 1.17088298, 1.55550789, 0.44383225, 0.37688275, -0.27983086, 1.77102953]) #thin
#shape = np.array([ 0.00404852, 0.8084637 , 0.32332591, -1.33163664, 1.05008727, 1.60955275, 0.22372946, -0.10738459, 0.89456312, -1.22231216]) #short
#shape = np.array([ 3.63453289, 1.20836171, 3.15674431, -0.78646793, -1.93847355, -0.32129994, -0.97771656, 0.94531640, 0.52825811, -0.99324327]) #tall
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname +
'_Pelvis'].location).copy()
data = cmu_parms[name]
fbegin = ishape * stepsize * stride
fend = min(ishape * stepsize * stride + stepsize * clipsize,
len(data['poses']))
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
matfile_info = join(
output_path,
name.replace(" ", "") + "_c%04d_info.mat" % (ishape + 1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
dict_info['bg'] = np.zeros((N, ), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty(3) # (1, 3)
dict_info['clipNo'] = ishape + 1
dict_info['cloth'] = np.zeros(
(N, ), dtype=np.object) # clothing texture image path
dict_info['gender'] = np.empty(N,
dtype='uint8') # 0 for male, 1 for female
dict_info['joints2D'] = np.empty(
(2, 24, N), dtype='float32') # 2D joint positions in pixel space
dict_info['joints3D'] = np.empty(
(3, 24, N), dtype='float32') # 3D joint positions in world coordinates
dict_info['light'] = np.empty((9, N), dtype='float32')
dict_info['pose'] = np.empty(
(data['poses'][0].size, N),
dtype='float32') # joint angles from SMPL (CMU)
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
# Note: Necessary for the HumANav dataset to canonically center the human.
dict_info['rightFootPos'] = np.empty((3, N), dtype='float32')
dict_info['rightToePos'] = np.empty((3, N), dtype='float32')
dict_info['leftFootPos'] = np.empty((3, N), dtype='float32')
dict_info['leftToePos'] = np.empty((3, N), dtype='float32')
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
if (output_types['vblur']):
dict_info['vblur_factor'] = np.empty(N, dtype='float32')
# for each clipsize'th frame in the sequence
get_real_frame = lambda ifr: ifr
random_zrot = 0
reset_loc = False
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob, obname,
scene, cam_ob,
smpl_data['regression_verts'],
smpl_data['joint_regressor'])
random_zrot = 2 * np.pi * np.random.rand()
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
# Needed to extracting toe and foot position & direction
with open('pkl/segm_per_v_overlap.pkl', 'rb') as f:
vsegm = load(f)
# Where the meshes and centering information is stored
base_dir = outdir
vs = np.arange(0, 1.85, .2)
# VS corresponds to the approximate velocity of the human (computer heuristically from mocap data), but after examining
# the actual data we decided to rebin the estimated human velocity for more realistic visual cues.
rebinned_vs = np.array([0., .2, .5, .6])
velocity_folders = make_velocity_dirs(base_dir, rebinned_vs)
pose_ishape_stride_str = 'pose_{:d}_ishape_{:d}_stride_{:d}'.format(
idx, ishape, stride)
body_shape_str = 'body_shape_{:d}'.format(body_shape_idx)
gender_str = gender
# create a keyframe animation with pose, translation, blendshapes and camera motion
# LOOP TO CREATE 3D ANIMATION
dt = 1. / 30.
prev_human_pos_3 = None
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob, obname,
scene, cam_ob, get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(
genders.values()).index(gender)]
if (output_types['vblur']):
dict_info['vblur_factor'][iframe] = vblur_factor
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
dict_info['zrot'][iframe] = random_zrot
scene.update()
# Bodies centered only in each minibatch of clipsize frames
if seq_frame == 0 or reset_loc:
reset_loc = False
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() -
orig_pelvis_loc.copy())
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
dict_info['camLoc'] = np.array(cam_ob.location)
# Compute the human (x, y, theta) position based on the location and orientation
# of its feet
human_pos_3 = compute_human_pos_3(scene, vsegm, ob)
human_speed = compute_human_speed(trans, prev_human_pos_3, dt=dt)
prev_human_pos_3 = trans
centering_data = {'human_pos_3': human_pos_3}
# If we can't estimate the speed from finite differencing (i.e. at the first timestep)
# do not save the mesh
if human_speed is not None:
print(human_speed)
# Compute the closest velocity bin and the corresponding folder name
# it will be None if the human_speed is not in the range [vmin, vmax]
velocity_folder = compute_velocity_folder(human_speed,
vs,
rebinned_vs,
velocity_folders,
vmax=1.85,
vmin=0.0)
if velocity_folder is not None:
human_data_output_folder = os.path.join(
velocity_folder, pose_ishape_stride_str, body_shape_str,
gender_str)
if not os.path.exists(human_data_output_folder):
os.makedirs(human_data_output_folder)
centeringFile = os.path.join(
human_data_output_folder,
'human_centering_info_{:d}.pkl'.format(seq_frame))
with open(centeringFile, 'wb') as f:
pickle.dump(centering_data, f)
# Exporting Human Mesh Here for Use in HumANav
meshFile = os.path.join(
human_data_output_folder,
'human_mesh_{:d}.obj'.format(seq_frame))
bpy.ops.export_scene.obj(filepath=meshFile,
keep_vertex_order=True,
group_by_object=True)
os._exit(0)
def main():
# time logging
global start_time
start_time = time.time()
import argparse
# parse commandline arguments
log_message(sys.argv)
parser = argparse.ArgumentParser(
description='Generate synth dataset images.')
parser.add_argument('--idx',
type=int,
help='idx of the requested sequence')
parser.add_argument('--ishape',
type=int,
help='requested cut, according to the stride')
parser.add_argument('--stride', type=int, help='stride amount, default 50')
parser.add_argument('--index',
type=int,
help='index of the texture in all.txt file')
parser.add_argument('--img_number',
type=int,
help='index of the image that is stored')
args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:])
idx = args.idx
ishape = args.ishape
stride = args.stride
index = args.index
img_number = args.img_number
log_message("input idx: %d" % idx)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
log_message("input index: %d" % index)
log_message("input img_number: %d" % img_number)
if idx is None:
exit(1)
if ishape is None:
exit(1)
if stride is None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# import idx info (name, split)
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
resy = params['resy']
resx = params['resx']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
# compute number of cuts
nb_ishape = max(
1, int(np.ceil(
(idx_info['nb_frames'] - (clipsize - stride)) / stride)))
log_message("Max ishape: %d" % (nb_ishape - 1))
if ishape is None:
exit(1)
assert (ishape < nb_ishape)
# name is set given idx
output_path = '/tmp'
tmp_path = '/mnt/hdd1/tanya/surreal_out/new_less_bright/Images'
# initialize RNG with seeds from sequence id
import hashlib
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10**
8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
if (output_types['vblur']):
vblur_factor = np.random.normal(0.5, 0.5)
params['vblur_factor'] = vblur_factor
log_message("Setup Blender")
# create copy-spher.harm. directory if not exists
sh_dir = join(tmp_path, 'spher_harm')
if not exists(sh_dir):
mkdir_safe(sh_dir)
sh_dst = join(sh_dir, 'sh_%02d_%05d.osl' % (runpass, idx))
os.system('cp spher_harm/sh.osl %s' % sh_dst)
genders = {0: 'female', 1: 'male'}
# pick random gender
gender = choice(genders)
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
log_message("Listing background images")
bg_names = join(bg_path, '%s_img.txt' % idx_info['use_split'])
nh_txt_paths = []
with open(bg_names) as f:
for line in f:
nh_txt_paths.append(join(bg_path, line))
# grab clothing names
log_message("clothing: %s" % clothing_option)
with open(join(smpl_data_folder, 'textures', 'all.txt')) as f:
txt_paths = f.read().splitlines()
txt_paths = sorted(txt_paths)
# if using only one source of clothing
# if clothing_option == 'nongrey':
# txt_paths = [k for k in txt_paths if 'nongrey' in k]
# elif clothing_option == 'grey':
# txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
cloth_img_name = txt_paths[index]
gender = cloth_img_name.split('/')[1]
cloth_img_name = join(smpl_data_folder, cloth_img_name)
cloth_img = bpy.data.images.load(cloth_img_name)
# random background
bg_img_name = choice(nh_txt_paths)[:-1]
bg_img = bpy.data.images.load(bg_img_name)
log_message("Loading parts segmentation")
beta_stds = np.load(join(smpl_data_folder, ('%s_beta_stds.npy' % gender)))
log_message("Building materials tree")
mat_tree = bpy.data.materials['Material'].node_tree
create_sh_material(mat_tree, sh_dst, cloth_img)
res_paths = create_composite_nodes(scene.node_tree,
params,
img=bg_img,
idx=idx)
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
camera_distance = np.random.normal(8.0, 0.01)
params['camera_distance'] = camera_distance
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = True # True: 0-24, False: expected to have 0-1 bg/fg
log_message("Creating materials segmentation")
# create material segmentation
if segmented_materials:
materials = create_segmentation(ob, params)
prob_dressed = {
'leftLeg': .5,
'leftArm': .9,
'leftHandIndex1': .01,
'rightShoulder': .8,
'rightHand': .01,
'neck': .01,
'rightToeBase': .9,
'leftShoulder': .8,
'leftToeBase': .9,
'rightForeArm': .5,
'leftHand': .01,
'spine': .9,
'leftFoot': .9,
'leftUpLeg': .9,
'rightUpLeg': .9,
'rightFoot': .9,
'head': .01,
'leftForeArm': .5,
'rightArm': .5,
'spine1': .9,
'hips': .9,
'rightHandIndex1': .01,
'spine2': .9,
'rightLeg': .5
}
else:
materials = {'FullBody': bpy.data.materials['Material']}
prob_dressed = {'FullBody': .6}
orig_pelvis_loc = (
arm_ob.matrix_world.copy() *
arm_ob.pose.bones[obname + '_Pelvis'].head.copy()) - Vector(
(-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data,
ob,
obname,
idx=idx,
gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
if idx_info['use_split'] == 'train':
fshapes = fshapes[:int(nb_fshapes * 0.8)]
elif idx_info['use_split'] == 'test':
fshapes = fshapes[int(nb_fshapes * 0.8):]
# pick random real body shape
shape = choice(fshapes) # +random_shape(.5) can add noise
# shape = random_shape(3.) # random body shape
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname +
'_Pelvis'].location).copy()
# create output directory
if not exists(output_path):
mkdir_safe(output_path)
# spherical harmonics material needs a script to be loaded and compiled
scs = []
for mname, material in materials.items():
scs.append(material.node_tree.nodes['Script'])
scs[-1].filepath = sh_dst
scs[-1].update()
rgb_path = tmp_path
data = cmu_parms[name]
fbegin = ishape * stepsize * stride
fend = min(ishape * stepsize * stride + stepsize * clipsize,
len(data['poses']))
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
nframes = len(data['poses'][::stepsize])
matfile_info = join(
output_path,
name.replace(" ", "") + "_c%04d_info.mat" % (ishape + 1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
dict_info['bg'] = np.zeros((N, ), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty(3) # (1, 3)
dict_info['clipNo'] = ishape + 1
# clothing texture image path
dict_info['cloth'] = np.zeros((N, ), dtype=np.object)
dict_info['gender'] = np.empty(N,
dtype='uint8') # 0 for male, 1 for female
# 2D joint positions in pixel space
dict_info['joints2D'] = np.empty((2, 24, N), dtype='float32')
# 3D joint positions in world coordinates
dict_info['joints3D'] = np.empty((3, 24, N), dtype='float32')
dict_info['light'] = np.empty((9, N), dtype='float32')
# joint angles from SMPL (CMU)
dict_info['pose'] = np.empty((data['poses'][0].size, N), dtype='float32')
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
if (output_types['vblur']):
dict_info['vblur_factor'] = np.empty(N, dtype='float32')
# for each clipsize'th frame in the sequence
def get_real_frame(ifr):
return ifr
random_zrot = 0
reset_loc = False
batch_it = 0
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob, obname,
scene, cam_ob,
smpl_data['regression_verts'],
smpl_data['joint_regressor'])
random_zrot = 2 * np.pi * np.random.rand()
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
# create a keyframe animation with pose, translation, blendshapes and
# camera motion
# LOOP TO CREATE 3D ANIMATION
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob, obname,
scene, cam_ob, get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(
genders.values()).index(gender)]
if (output_types['vblur']):
dict_info['vblur_factor'][iframe] = vblur_factor
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
dict_info['zrot'][iframe] = random_zrot
scene.update()
# Bodies centered only in each minibatch of clipsize frames
if seq_frame == 0 or reset_loc:
reset_loc = False
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() -
orig_pelvis_loc.copy())
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
dict_info['camLoc'] = np.array(cam_ob.location)
scene.node_tree.nodes['Image'].image = bg_img
for part, material in materials.items():
material.node_tree.nodes['Vector Math'].inputs[1].default_value[:2] = (
0, 0)
'''
# random light
sh_coeffs = .7 * (2 * np.random.rand(9) - 1)
# Ambient light (first coeff) needs a minimum is ambient. Rest is
# uniformly distributed, higher means brighter.
sh_coeffs[0] = .5 + .9 * np.random.rand()
sh_coeffs[1] = -.7 * np.random.rand()
'''
sh_coeffs = .3 * np.ones(9)
# sh_coeffs[1:] = 0.01
for ish, coeff in enumerate(sh_coeffs):
for sc in scs:
sc.inputs[ish + 1].default_value = coeff
# iterate over the keyframes and render
# LOOP TO RENDER
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
scene.frame_set(get_real_frame(seq_frame))
iframe = seq_frame
dict_info['bg'][iframe] = bg_img_name
dict_info['cloth'][iframe] = cloth_img_name
dict_info['light'][:, iframe] = sh_coeffs
scene.render.use_antialiasing = False
scene.render.filepath = join(
rgb_path, 'Image_{}.png'.format(img_number + seq_frame))
log_message("Rendering frame %d" % seq_frame)
log_message("Rendering image {}".format(rgb_path))
# disable render output
logfile = '/dev/null'
open(logfile, 'a').close()
old = os.dup(1)
sys.stdout.flush()
os.close(1)
os.open(logfile, os.O_WRONLY)
# Render
bpy.ops.render.render(write_still=True)
# disable output redirection
os.close(1)
os.dup(old)
os.close(old)
# bone locations should be saved after rendering so that the bones are
# updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene,
cam_ob)
dict_info['joints2D'][:, :, iframe] = np.transpose(bone_locs_2D)
dict_info['joints3D'][:, :, iframe] = np.transpose(bone_locs_3D)
reset_loc = (bone_locs_2D.max(axis=-1) >
256).any() or (bone_locs_2D.min(axis=0) < 0).any()
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = \
Quaternion((1, 0, 0, 0))
def gen_one_seqs(setting, rand_noise=True):
"""
Note that human shape only has a slight motion within each sequences,
try to use a larger stepsize
TODO:
1. no scaling, as I removed the orig_trans, and each step translation
2. shape and pose are randomly noised
"""
## setup basic variables
split = setting.split
gender = setting.gender
idx_shape = setting.ishape
idx_seq = setting.iseq
idx_run = setting.irun
idx_cloth = setting.icloth
idx_bg = setting.ibg
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_root = params['tmp_path']
out_path = params['output_path']
stepsize = params['stepsize']
# motion blur
params["vblur"] = np.random.normal(0.5, 0.5)
for k in ["image", "depth", "mat", "mesh"]:
if not os.path.exists(out_path + "/" + k + "/"):
mkdir_safe(out_path + "/" + k + "/")
# check if already computed
# + clean up existing tmp folders if any
# identifier (run_$split_$gender_$idxrun_$idxseq_$idxshape_$idxframe)
identifier = "_".join([
"image", split, gender,
"%03d" % idx_run,
"%04d" % idx_seq,
"%04d" % idx_shape
])
tmp_path = join(tmp_root, identifier)
if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
os.system('rm -rf %s' % tmp_path)
if not exists(tmp_path):
mkdir_safe(tmp_path)
res_paths = {
k: join(tmp_root, identifier.replace("image", k))
for k in params['output_types'] if params['output_types'][k]
}
# create copy-spher.harm. directory if not exists
sh_dir = join(tmp_path, 'spher_harm')
if not exists(sh_dir):
mkdir_safe(sh_dir)
sh_dst = join(sh_dir, 'sh.osl')
os.system('cp spher_harm/sh.osl %s' % sh_dst)
## reset
bpy.ops.wm.read_factory_settings()
### setup blender
log_message("Setup blender")
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
## load all bg images
log_message("Listing background images")
bg_names = join(bg_path, '%s_img.txt' % split)
nh_txt_paths = []
with open(bg_names) as f:
for line in f:
nh_txt_paths.append(join(bg_path, line))
## grab clothing names
clothing_option = "nongrey"
log_message("clothing: %s" % clothing_option)
with open(join(smpl_data_folder, 'textures',
'%s_%s.txt' % (gender, split))) as f:
txt_paths = f.read().splitlines()
txt_paths = [k for k in txt_paths if clothing_option in k]
# random clothing texture
idx_cloth = idx_cloth % len(txt_paths)
cloth_img_name = txt_paths[idx_cloth]
cloth_img_name = join(smpl_data_folder, cloth_img_name)
cloth_img = bpy.data.images.load(cloth_img_name)
# random background
idx_bg = idx_bg % len(nh_txt_paths)
bg_img_name = nh_txt_paths[idx_bg][:-1]
bg_img = bpy.data.images.load(bg_img_name)
## load smpl motion data
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
## start to setup the scene
log_message("Building materials tree")
mat_tree = bpy.data.materials['Material'].node_tree
create_sh_material(mat_tree, sh_dst, cloth_img)
create_composite_nodes(scene.node_tree,
params,
res_paths,
img=bg_img,
idx=idx_seq)
log_message("Initializing scene")
set_salient(1)
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = True #True: 0-24, False: expected to have 0-1 bg/fg
set_salient(0)
# create material segmentation
log_message("Creating materials segmentation")
if segmented_materials:
materials = create_segmentation(ob, params)
else:
materials = {'FullBody': bpy.data.materials['Material']}
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
# load shape, pose for the given sequence index.
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data,
ob,
obname,
idx=idx_seq,
gender=gender)
# pick random real body shape
log_message("Loaded body data for %s" % name)
if split == 'train':
fshapes = fshapes[:int(len(fshapes) * 0.6)]
elif split == 'val':
fshapes = fshapes[int(len(fshapes) * 0.6):int(len(fshapes) * 0.8)]
elif split == 'test':
fshapes = fshapes[int(len(fshapes) * 0.8):]
shape = fshapes[idx_shape %
(len(fshapes))] #+random_shape(.5) can add noise
## setup the pelvis to origin for ease
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname +
'_Pelvis'].location).copy()
# create output directory
if not exists(out_path):
mkdir_safe(out_path)
# spherical harmonics material needs a script to be loaded and compiled
scs = []
for mname, material in materials.items():
scs.append(material.node_tree.nodes['Script'])
scs[-1].filepath = sh_dst
scs[-1].update()
data = cmu_parms[name]
fbegin = 0
fend = len(data['poses']) // 50 * 50
stepsize = len(data['poses']) // 50
# stepsize*min(stepsize, 150)
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
## clearup, and put the origin at zeros
orig_pelvis_loc = (arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy())\
- Vector((-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
reset_joint_positions(Vector((0, 0, 0)), shape, ob, arm_ob, obname, scene,
cam_ob, smpl_data['regression_verts'],
smpl_data['joint_regressor'])
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
### lighting and rendering parameters
scene.node_tree.nodes['Image'].image = bg_img
## lighting random:
sh_coeffs = .7 * (2 * np.random.rand(9) - 1)
# Ambient light (first coeff) needs a minimum is ambient. Rest is uniformly distributed, higher means brighter.
sh_coeffs[0] = .5 + .9 * np.random.rand()
sh_coeffs[1] = -.7 * np.random.rand()
# TODO: what's the function??
for part, material in materials.items():
material.node_tree.nodes['Vector Math'].inputs[1].default_value[:2] = (
0, 0)
for ish, coeff in enumerate(sh_coeffs):
for sc in scs:
sc.inputs[ish + 1].default_value = coeff
get_real_frame = lambda ifr: ifr
shape_orig = shape
# iterate over the keyframes and render
# LOOP TO RENDER
render_path = res_paths["image"]
for seq_frame, (pose, trans) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
## random set pose or shape
pose = pose + (np.random.rand(len(pose)) * 2 - 1) * 0.1 * rand_noise
shape = shape_orig + (np.random.rand(10) * 2 - 1) * 0.05 * rand_noise
# disable render output
set_salient(1)
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
# shape at the origin, and translate the camera inversly
apply_trans_pose_shape(Vector((0, 0, 0)), pose, shape, ob,
arm_ob, obname, scene, cam_ob,
get_real_frame(seq_frame))
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
Euler((0, 0, 0), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
scene.update()
if seq_frame == 0:
# Bodies centered only in each minibatch of clipsize frames
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() -
orig_pelvis_loc.copy())
# cam_ob.location = orig_cam_loc.copy() - Vector(trans)
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
# Render the view from all given cameras
log_message("Rendering frame %d" % seq_frame)
scene.render.use_antialiasing = False
res_paths[
"image"] = render_path + "_%04d.png" % get_real_frame(seq_frame)
scene.render.filepath = res_paths["image"]
bpy.ops.render.render(write_still=True)
# disable output redirection
set_salient(0)
# each frame data as a mat file
## save key information (bg, cloth, light, shape, pose, vblur, gender, split)
dict_info = {}
dict_info['bg'] = bg_img_name
dict_info['cloth'] = cloth_img_name
dict_info['light'] = sh_coeffs
dict_info['cam_loc'] = np.array(cam_ob.location)
dict_info['pose'] = pose
dict_info['shape'] = shape
dict_info['split'] = split
dict_info['gender'] = gender
dict_info['seqs'] = np.array([idx_seq, idx_shape, stepsize], np.int32)
dict_info['vblur'] = params["vblur"]
dict_info['camera_K'] = np.array(
get_calibration_matrix_K_from_blender(cam_ob.data))
dict_info['camera_matrix_world'] = np.array(cam_ob.matrix_world)
R, T = get_RT_matrix_from_blender(cam_ob)
dict_info['camera_R'] = np.array(R)
dict_info['camera_T'] = np.array(T)
dict_info['pelvis_loc'] = np.array(
arm_ob.matrix_world.copy() *
arm_ob.pose.bones[obname + '_Pelvis'].head.copy())
# NOTE:
# ideally, pixels should be readable from a viewer node, but I get only zeros
# --> https://ammous88.wordpress.com/2015/01/16/blender-access-render-results-pixels-directly-from-python-2/
# len(np.asarray(bpy.data.images['Render Result'].pixels) is 0
# Therefore we write them to temporary files and read with OpenEXR library (available for python2) in main_part2.py
# Alternatively, if you don't want to use OpenEXR library, the following commented code does loading with Blender functions, but it can cause memory leak.
# If you want to use it, copy necessary lines from main_part2.py such as definitions of dict_normal, matfile_normal...
## TODO: where to store???
## list(image, depth, seg, normal, gt_flow, vblur, fg)
imagefile_path = join(
out_path,
"image/" + identifier + "_%04d.png" % get_real_frame(seq_frame))
for k, folder in res_paths.items():
if not k == 'vblur' and not k == 'fg':
filename = "Image%04d.exr" % get_real_frame(seq_frame)
if k == "image":
render_img = bpy.data.images.load(folder)
else:
render_img = bpy.data.images.load(folder + "/" + filename)
# render_img.pixels size is width * height * 4 (rgba)
arr = np.array(render_img.pixels[:]).reshape(
(params["resx"], params["resy"],
-1))[::-1, :, :] # images are vertically flipped
if k == 'normal': # 3 channels, original order
mat = arr[:, :, :3]
elif k == 'gtflow':
mat = arr[:, :, 1:3]
elif k == 'depth':
mat = arr[:, :, 0]
mat_c = mat.copy()
depth_m = sorted(mat_c.flatten())[-2]
mat_c[mat_c > depth_m] = depth_m + 1.0
cv2.imwrite(imagefile_path.replace("image", "depth"),
255.0 / (mat_c + 1e-5))
elif k == 'segm':
mat = arr[:, :, 0]
elif k == "image":
mat = arr[:, :, :3]
cv2.imwrite(imagefile_path, mat * 255)
if (k in params['output_types']):
dict_info[k] = mat
# remove the image to release memory, object handles, etc.
render_img.user_clear()
bpy.data.images.remove(render_img)
dict_info['image'] = cv2.imread(scene.render.filepath)
# bone locations should be saved after rendering so that the bones are updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene,
cam_ob)
dict_info['joints2D'] = np.transpose(bone_locs_2D)
dict_info['joints3D'] = np.transpose(bone_locs_3D)
## export rendering, parameters, and meshes to the output directory
log_message("Export parameters, images, and meshes for " +
imagefile_path)
dd.io.save(imagefile_path.replace("image", "mat"), dict_info)
# export human body
bpy.ops.object.select_all(action='DESELECT')
ob.select = True
bpy.ops.export_scene.obj(filepath=imagefile_path.replace(
"image", "mesh").replace(".png", ".obj"),
use_selection=True,
use_uvs=False,
use_materials=False)
bpy.ops.object.select_all(action='DESELECT')
# TODO: ????
arm_ob.pose.bones[obname + '_root'].rotation_quaternion = Quaternion(
(1, 0, 0, 0))
## clear up the imported objects, as we import them each loading.
bpy.ops.object.select_all(action='DESELECT')
ob.select = True
arm_ob.select = True
bpy.ops.object.delete(use_global=False)
return fend // stepsize + 1
def main(runpass=None, idx=None, idx_info=None, stride=None, cmu_idx=0):
# time logging
global start_time
start_time = time.time()
import argparse
#idx = 3
#ishape = 0
#stride = 50
# # parse commandline arguments
# log_message(sys.argv)
# parser = argparse.ArgumentParser(description='Generate synth dataset images.')
# parser.add_argument('--idx', type=int,
# help='idx of the requested sequence')
# parser.add_argument('--ishape', type=int,
# help='requested cut, according to the stride')
# parser.add_argument('--stride', type=int,
# help='stride amount, default 50')
#
# args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:])
#
# idx = args.idx
# ishape = args.ishape
# stride = args.stride
#if runpass == None:
# log_message("WARNING: runpass not specified, using default value 0")
# runpass = 0
#if idx == None:
# log_message("WARNING: idx not specified, using default value 0")
# idx = 0
#if stride == None:
# log_message("WARNING: stride not specified, using default value 50")
# stride = 50
log_message("input idx: %d" % idx)
log_message("input stride: %d" % stride)
# import idx info (name, split)
#idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
#(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
resy = params['resy']
resx = params['resx']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
#openexr_py2_path = params['openexr_py2_path']
# compute number of cuts
#nb_ishape = max(1, int(np.ceil((idx_info['nb_frames'] - (clipsize - stride))/stride)))
nb_ishape = 1
log_message("Max ishape: %d" % (nb_ishape - 1))
#if ishape == None:
# exit(1)
#assert(ishape < nb_ishape)
# name is set given idx
name = idx_info['name']
#output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
#params['output_path'] = output_path
#tmp_path = join(tmp_path, 'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
#params['tmp_path'] = tmp_path
# check if already computed
# + clean up existing tmp folders if any
#if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
# os.system('rm -rf %s' % tmp_path)
#rgb_vid_filename = "%s_c%04d.mp4" % (join(output_path, name.replace(' ', '')), (ishape + 1))
#if os.path.isfile(rgb_vid_filename):
# log_message("ALREADY COMPUTED - existing: %s" % rgb_vid_filename)
# return 0
# create tmp directory
#if not exists(tmp_path):
# mkdir_safe(tmp_path)
# >> don't use random generator before this point <<
# initialize RNG with seeds from sequence id
#import hashlib
#s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
#seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10 ** 8)
#log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
#random.seed(seed_number)
#np.random.seed(seed_number)
#if(output_types['vblur']):
# vblur_factor = np.random.normal(0.5, 0.5)
# params['vblur_factor'] = vblur_factor
#log_message("Setup Blender")
# create copy-spher.harm. directory if not exists
#sh_dir = join(tmp_path, 'spher_harm')
#if not exists(sh_dir):
# mkdir_safe(sh_dir)
#sh_dst = join(sh_dir, 'sh_%02d_%05d.osl' % (runpass, idx))
#os.system('cp spher_harm/sh.osl %s' % sh_dst)
genders = {0: 'female', 1: 'male'}
# pick random gender
#gender = choice(genders)
for ishape in range(nb_ishape):
#for ishape in range(1):
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(),
16) % (10**8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
gender = choice(genders)
#gender = 'male'
output_path = join(output_path, 'run%d' % runpass,
name.replace(" ", "") + "_%s" % (gender))
params['output_path'] = output_path
log_message("current ishape: %d" % ishape)
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
#log_message("Listing background images")
#bg_names = join(bg_path, '%s_img.txt' % idx_info['use_split'])
#nh_txt_paths = []
#with open(bg_names) as f:
# for line in f:
# nh_txt_paths.append(join(bg_path, line))
# grab clothing names
#log_message("clothing: %s" % clothing_option)
#with open( join(smpl_data_folder, 'textures', '%s_%s.txt' % ( gender, idx_info['use_split'] ) ) ) as f:
# txt_paths = f.read().splitlines()
# if using only one source of clothing
#if clothing_option == 'nongrey':
# txt_paths = [k for k in txt_paths if 'nongrey' in k]
#elif clothing_option == 'grey':
# txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
#cloth_img_name = choice(txt_paths)
#cloth_img_name = join(smpl_data_folder, cloth_img_name)
#cloth_img = bpy.data.images.load(cloth_img_name)
# random background
#bg_img_name = choice(nh_txt_paths)[:-1]
#bg_img = bpy.data.images.load(bg_img_name)
log_message("Loading parts segmentation")
beta_stds = np.load(
join(smpl_data_folder, ('%s_beta_stds.npy' % gender)))
log_message("Building materials tree")
# mat_tree = bpy.data.materials['Material'].node_tree
# create_sh_material(mat_tree, sh_dst, cloth_img)
# res_paths = create_composite_nodes(scene.node_tree, params, img=bg_img, idx=idx) # Only human model is saved in this step, no background
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
#camera_distance = np.random.normal(8.0, 1)
camera_distance = CAMERA_DISTANCE
params[
'camera_distance'] = camera_distance ###############################################################
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = False #True: 0-24, False: expected to have 0-1 bg/fg
log_message("Creating materials segmentation")
# create material segmentation
if segmented_materials:
materials = create_segmentation(ob, params)
prob_dressed = {
'leftLeg': .5,
'leftArm': .9,
'leftHandIndex1': .01,
'rightShoulder': .8,
'rightHand': .01,
'neck': .01,
'rightToeBase': .9,
'leftShoulder': .8,
'leftToeBase': .9,
'rightForeArm': .5,
'leftHand': .01,
'spine': .9,
'leftFoot': .9,
'leftUpLeg': .9,
'rightUpLeg': .9,
'rightFoot': .9,
'head': .01,
'leftForeArm': .5,
'rightArm': .5,
'spine1': .9,
'hips': .9,
'rightHandIndex1': .01,
'spine2': .9,
'rightLeg': .5
}
else:
materials = {'FullBody': bpy.data.materials['Material']}
prob_dressed = {'FullBody': .6}
# orig_pelvis_loc = (arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy()) - Vector((-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data,
ob,
obname,
idx=cmu_idx,
gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
if idx_info['use_split'] == 'train':
fshapes = fshapes[:int(nb_fshapes * 0.8)]
elif idx_info['use_split'] == 'test':
fshapes = fshapes[int(nb_fshapes * 0.8):]
# pick random real body shape
random.seed(seed_number)
shape = choice(fshapes) #+random_shape(.5) can add noise
# example shapes
#shape = np.zeros(10) #average
#shape = np.array([ 2.25176191, -3.7883464 , 0.46747496, 3.89178988, 2.20098416, 0.26102114, -3.07428093, 0.55708514, -3.94442258, -2.88552087]) #fat
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname +
'_Pelvis'].location).copy()
# create output directory
if not exists(output_path):
mkdir_safe(output_path)
# spherical harmonics material needs a script to be loaded and compiled
# scs = []
# for mname, material in materials.items():
# scs.append(material.node_tree.nodes['Script'])
# scs[-1].filepath = sh_dst
# scs[-1].update()
#rgb_dirname = name.replace(" ", "") + '_c%04d.mp4' % (ishape + 1)
#rgb_path = join(tmp_path, rgb_dirname)
data = cmu_parms[name]
#fbegin = ishape*stepsize*stride
#fend = min(ishape*stepsize*stride + stepsize*clipsize, len(data['poses']))
fbegin = 0
fend = len(data['poses'])
log_message("Computing how many frames to allocate")
#N = len(data['poses'][fbegin:fend:stepsize])
N = len(data['poses'][fbegin:fend:stepsize]) + PREAMBLE_FRAME_NUM
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
nframes = len(data['poses'][::stepsize])
matfile_info = join(
output_path,
name.replace(" ", "") + "_c%04d_info.mat" % (ishape + 1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
# dict_info['bg'] = np.zeros((N,), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty((3, N), dtype='float32') # (1, 3)
dict_info['clipNo'] = ishape + 1
# dict_info['cloth'] = np.zeros((N,), dtype=np.object) # clothing texture image path
dict_info['gender'] = np.empty(
N, dtype='uint8') # 0 for male, 1 for female
dict_info['joints2D'] = np.empty(
(2, 24, N), dtype='float32') # 2D joint positions in pixel space
dict_info['joints3D'] = np.empty(
(3, 24, N),
dtype='float32') # 3D joint positions in world coordinates
# dict_info['light'] = np.empty((9, N), dtype='float32')
dict_info['pose'] = np.empty(
(data['poses'][0].size, N),
dtype='float32') # joint angles from SMPL (CMU)
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
# dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
dict_info['seednumber'] = seed_number
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
#if(output_types['vblur']):
# dict_info['vblur_factor'] = np.empty(N, dtype='float32') #############################################
# for each clipsize'th frame in the sequence
get_real_frame = lambda ifr: ifr
#random_zrot = 0
reset_loc = False
batch_it = 0
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob,
obname, scene, cam_ob,
smpl_data['regression_verts'],
smpl_data['joint_regressor'])
#random_zrot = 2*np.pi*np.random.rand()
arm_ob.animation_data_clear()
cam_ob.animation_data_clear(
) #####################################################################################################
# create a keyframe animation with pose, translation, blendshapes and camera motion
# LOOP TO CREATE 3D ANIMATION
# rotate model to front up-right
#arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion(Euler((math.radians(-90), 0, 0), 'XYZ'))
#JIANL: Weired! Weired! Weired!
#JIANL: The value is different for system-terminal and blender's python console (if the rotation was applied to "root")
#JIANL: To solve this issue, modify arm_ob.matrix_world instead!
# This is to make local coordinate the same as global coordinate
arm_ob.matrix_world = Matrix(
((1.0000, 0.0000, 0.0000, 0.0000), (0.0000, 1.0000, 0.0000,
0.0000),
(0.0000, 0.0000, 1.0000, 0.0000), (0.0000, 0.0000, 0.0000,
1.0000)))
curr_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
# After resetting the local coordinate, curr_pelvis_loc[1] is y-axis value,
# curr_pelvis_loc[2] is the z-axis value,
# [math.radians(90), 0, 0] rotation to get T-pose
# so, move curr_pelvis_loc[1] along y-axis (negative)
# and move (curr_pelvis_loc[1]-curr_pelvis_loc[2]) along z-axis (positive)
# this will result in a right pelvis position, which makes foot touching the ground
TRANS_OFFSET = [
0., curr_pelvis_loc[1], -(curr_pelvis_loc[1] - curr_pelvis_loc[2])
] # OFFSET to make model stand on ground at origin
#print(TRANS_OFFSET)
# interpolate frames to add T-pose transition
for seq_frame in range(PREAMBLE_FRAME_NUM):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
#trans = data['trans'][fbegin:fend:stepsize][0].copy()
#trans = np.zeros(3) - [0., 0.91, 0.]
trans = np.zeros(
3) - TRANS_OFFSET # make model stand on ground at origin
pose = data['poses'][fbegin:fend:stepsize][0].copy()
pose[0] = math.radians(90) + seq_frame * (
(pose[0] - math.radians(90)) / PREAMBLE_FRAME_NUM)
pose[1] = seq_frame * (pose[1] / PREAMBLE_FRAME_NUM
) # interpolate rotation
pose[2] = seq_frame * (pose[2] / PREAMBLE_FRAME_NUM
) # interpolate rotation
pose[3:] = seq_frame * (pose[3:] / PREAMBLE_FRAME_NUM
) # interpolate rotation
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob,
obname, scene, cam_ob,
get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(
genders.values()).index(gender)]
#if(output_types['vblur']):
# dict_info['vblur_factor'][iframe] = vblur_factor
#arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion(Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
#dict_info['zrot'][iframe] = random_zrot
scene.update()
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
# keep camera tracking the model on a plane, modify if axis locking is required
# camera location is recorded for rendering
dict_info['camLoc'][:, iframe] = orig_cam_loc.copy(
) + new_pelvis_loc - Vector([
0, 0, 1
]) #####################################################
for seq_frame_, (pose_, trans_) in enumerate(
zip(data['poses'][fbegin:fend:stepsize],
data['trans'][fbegin:fend:stepsize])):
seq_frame = seq_frame_ + PREAMBLE_FRAME_NUM
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
pose = pose_.copy()
#trans = trans_.copy() - data['trans'][fbegin:fend:stepsize][0] - [0., 0.91, 0.]
trans = trans_.copy() - data['trans'][fbegin:fend:stepsize][
0] - TRANS_OFFSET # make model stand on ground at origin
#pose[0:3] = 0
#trans[:] = trans[[0,2,1]]
#print(trans)
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob,
obname, scene, cam_ob,
get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(
genders.values()).index(gender)]
#if(output_types['vblur']):
# dict_info['vblur_factor'][iframe] = vblur_factor
#arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion(Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname + '_root'].keyframe_insert(
'rotation_quaternion', frame=get_real_frame(seq_frame))
#dict_info['zrot'][iframe] = random_zrot
scene.update()
#print(seq_frame)
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[
obname + '_Pelvis'].head.copy()
dict_info['camLoc'][:, iframe] = orig_cam_loc.copy(
) + new_pelvis_loc - Vector([
0, 0, 1
]) #####################################################
matfile_info = join(
output_path,
name.replace(" ", "") + "_c%04d_info.mat" % (ishape + 1))
# SAVE DATASET
# save .obj for human model
scene.frame_set(get_real_frame(0))
obname = '%s_avg' % gender[0]
ob = bpy.data.objects[obname]
scn = bpy.context.scene
scn.objects.active = ob
#obj_filename = name + '_' + gender + '.obj'
obj_body_saved = join(
output_path,
name.replace(" ", "") + "_c%04d_%s.obj" % (ishape + 1, gender))
bpy.ops.export_scene.obj(filepath=obj_body_saved,
check_existing=True,
axis_forward='-Z',
axis_up='Y',
filter_glob="*.obj;*.mtl",
use_selection=True,
use_animation=False,
use_mesh_modifiers=True,
use_edges=True,
use_smooth_groups=False,
use_smooth_groups_bitflags=False,
use_normals=True,
use_uvs=True,
use_materials=False,
use_triangles=False,
use_nurbs=False,
use_vertex_groups=False,
use_blen_objects=True,
group_by_object=False,
group_by_material=False,
keep_vertex_order=False,
global_scale=1,
path_mode='AUTO')
# save .mdd for animation
#mdd_filename = name + '_' + gender + '.mdd'
mdd_body_saved = join(
output_path,
name.replace(" ", "") + "_c%04d_%s.mdd" % (ishape + 1, gender))
# to align .mdd to .fbx, frame_start is set to 0, and use_rest_frame is true
bpy.ops.export_shape.mdd(filepath=mdd_body_saved,
check_existing=True,
filter_glob="*.mdd",
fps=25,
frame_start=0,
frame_end=N,
use_rest_frame=True)
# save .fbx for animation
#fbx_filename = name + '_' + gender + '.fbx'
fbx_body_saved = join(
output_path,
name.replace(" ", "") + "_c%04d_%s.fbx" % (ishape + 1, gender))
# export "ARMATURE" only to save space
bpy.ops.export_scene.fbx(filepath=fbx_body_saved,
check_existing=True,
axis_forward='-Z',
axis_up='Y',
filter_glob="*.fbx",
version='BIN7400',
ui_tab='MAIN',
use_selection=False,
global_scale=1,
apply_unit_scale=True,
bake_space_transform=False,
object_types={'ARMATURE'},
use_mesh_modifiers=False,
mesh_smooth_type='OFF',
use_mesh_edges=False,
use_tspace=False,
use_custom_props=False,
add_leaf_bones=True,
primary_bone_axis='Y',
secondary_bone_axis='X',
use_armature_deform_only=False,
armature_nodetype='NULL',
bake_anim=True,
bake_anim_use_all_bones=True,
bake_anim_use_nla_strips=True,
bake_anim_use_all_actions=True,
bake_anim_force_startend_keying=True,
bake_anim_step=1,
bake_anim_simplify_factor=1,
use_anim=True,
use_anim_action_all=True,
use_default_take=True,
use_anim_optimize=True,
anim_optimize_precision=6,
path_mode='AUTO',
embed_textures=False,
batch_mode='OFF',
use_batch_own_dir=True,
use_metadata=True)
# iterate over the keyframes and record annotations
# LOOP TO ANNOTATE
#for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
for seq_frame in range(N):
scene.frame_set(get_real_frame(seq_frame))
iframe = seq_frame
# dict_info['bg'][iframe] = bg_img_name
# dict_info['cloth'][iframe] = cloth_img_name
# dict_info['light'][:, iframe] = sh_coeffs
# scene.render.use_antialiasing = False
# scene.render.filepath = join(rgb_path, 'Image%04d.png' % get_real_frame(seq_frame))
log_message("Annotating frame %d" % seq_frame)
# # disable render output
# logfile = '/dev/null'
# open(logfile, 'a').close()
# old = os.dup(1)
# sys.stdout.flush()
# os.close(1)
# os.open(logfile, os.O_WRONLY)
# # Render
# bpy.ops.render.render(write_still=True)
# # disable output redirection
# os.close(1)
# os.dup(old)
# os.close(old)
# bone locations should be saved after rendering so that the bones are updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene,
cam_ob)
dict_info['joints2D'][:, :, iframe] = np.transpose(bone_locs_2D)
dict_info['joints3D'][:, :, iframe] = np.transpose(bone_locs_3D)
#reset_loc = (bone_locs_2D.max(axis=-1) > 256).any() or (bone_locs_2D.min(axis=0) < 0).any()
#arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion((1, 0, 0, 0))
# save a .blend file for debugging:
# bpy.ops.wm.save_as_mainfile(filepath=join(tmp_path, 'pre.blend'))
# save annotation excluding png/exr data to _info.mat file
import scipy.io
scipy.io.savemat(matfile_info, dict_info, do_compression=True)
# Initialize Blender for the next clip generation
bpy.ops.wm.read_homefile()
import os
import shutil
import sys
sys.path.insert(0, ".")
female_train_filename = 'txt/female_train_textures.txt'
male_train_filename = 'txt/male_train_textures.txt'
female_test_filename = 'txt/female_test_textures.txt'
male_test_filename = 'txt/male_test_textures.txt'
base_outdir = 'human_textures'
import config
params = config.load_file('config_local', 'SYNTH_DATA')
smpl_data_dir = params['smpl_data_folder']
def mkdir_if_missing(dirname):
if not os.path.exists(dirname):
os.makedirs(dirname)
def copy_human_textures(mode, gender, filename):
outdir = os.path.join(base_outdir, mode, gender)
mkdir_if_missing(outdir)
with open(filename, 'r') as f:
textures = f.read().split('\n')
# Append base directory to each texture
def main_part2(file_cfg):
# time logging
global start_time
start_time = time.time()
# param list:
with open(file_cfg, "rb") as f:
print("load configs from: " + file_cfg)
config = pickle.load(f)
seed_number = config.seed_number
idx_gender = config.idx_gender
idx_bg = config.idx_bg
idx_fshape = config.idx_fshape
idx_cloth = config.idx_cloth
disp_bg = config.disp_bg
# human data source:
idx = config.idx_seq
ishape = config.idx_ishape
stride = config.stride
from pickle import load
import argparse
log_message("input idx: %d" % idx)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
if idx == None:
exit(1)
if ishape == None:
exit(1)
if stride == None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# import idx info (name, split)
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("start part 2")
import hashlib
import random
# initialize random seeds with sequence id
s = "synth_data:%d:%d:%d" % (idx, runpass, ishape)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
# import configuration
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
resy = params['resy']
resx = params['resx']
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
name = idx_info[idx]["name"]
# compute number of cuts
nb_ishape = max(1, int(np.ceil((idx_info[idx]['nb_frames'] - (clipsize - stride))/stride)))
ishape = ishape%nb_ishape
output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
log_message("output path: " + output_path)
tmp_path = join(tmp_path, 'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
# check whether openexr_py2_path is loaded from configuration file
if 'openexr_py2_path' in locals() or 'openexr_py2_path' in globals():
for exr_path in openexr_py2_path.split(':'):
sys.path.insert(1, exr_path)
# to read exr imgs
import OpenEXR
import array
import Imath
log_message("Loading SMPL data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
cmu_parms, name = load_body_data(smpl_data, idx)
res_paths = {k:join(tmp_path, '%05d_%s'%(idx, k)) for k in output_types if output_types[k]}
data = cmu_parms[name]
nframes = len(data['poses'][::stepsize])
# .mat files
matfile_normal = join(output_path, name.replace(" ", "") + "_c%04d_normal.mat" % (ishape + 1))
matfile_gtflow = join(output_path, name.replace(" ", "") + "_c%04d_gtflow.mat" % (ishape + 1))
matfile_depth = join(output_path, name.replace(" ", "") + "_c%04d_depth.mat" % (ishape + 1))
matfile_segm = join(output_path, name.replace(" ", "") + "_c%04d_segm.mat" % (ishape + 1))
dict_normal = {}
dict_gtflow = {}
dict_depth = {}
dict_segm = {}
get_real_frame = lambda ifr: ifr
FLOAT = Imath.PixelType(Imath.PixelType.FLOAT)
# overlap determined by stride (# subsampled frames to skip)
fbegin = ishape*stepsize*stride
fend = min(ishape*stepsize*stride + stepsize*clipsize, len(data['poses']))
# LOOP OVER FRAMES
for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
log_message("Processing frame %d" % iframe)
for k, folder in res_paths.items():
if not k== 'vblur' and not k=='fg':
for ii in range(2):
path = join(folder, 'Image%04d_%d.exr' % (get_real_frame(seq_frame), ii))
exr_file = OpenEXR.InputFile(path)
if k == 'normal':
mat = np.transpose(np.reshape([array.array('f', exr_file.channel(Chan, FLOAT)).tolist() for Chan in ("R", "G", "B")], (3, resx, resy)), (1, 2, 0))
dict_normal['normal_%04d_%01d' % (iframe + 1, ii+1)] = mat.astype(np.float32, copy=False) # +1 for the 1-indexing
elif k == 'gtflow':
mat = np.transpose(np.reshape([array.array('f', exr_file.channel(Chan, FLOAT)).tolist() for Chan in ("R", "G")], (2, resx, resy)), (1, 2, 0))
dict_gtflow['gtflow_%04d_%01d' % (iframe + 1, ii+1)] = mat.astype(np.float32, copy=False)
elif k == 'depth':
mat = np.reshape([array.array('f', exr_file.channel(Chan, FLOAT)).tolist() for Chan in ("R")], (resx, resy))
dict_depth['depth_%04d_%01d' % (iframe + 1, ii+1)] = mat.astype(np.float32, copy=False)
elif k == 'segm':
mat = np.reshape([array.array('f', exr_file.channel(Chan, FLOAT)).tolist() for Chan in ("R")], (resx, resy))
dict_segm['segm_%04dd_%01d' % (iframe + 1, ii+1)] = mat.astype(np.uint8, copy=False)
# remove(path)
print("render infos: ")
print(res_paths)
print("#normal: %d"%(len(dict_normal.keys())))
print("#depth: %d"%(len(dict_depth.keys())))
print("#segm: %d"%(len(dict_segm.keys())))
import scipy.io
scipy.io.savemat(matfile_normal, dict_normal, do_compression=True)
scipy.io.savemat(matfile_gtflow, dict_gtflow, do_compression=True)
scipy.io.savemat(matfile_depth, dict_depth, do_compression=True)
scipy.io.savemat(matfile_segm, dict_segm, do_compression=True)
# cleaning up tmp
if tmp_path != "" and tmp_path != "/":
log_message("Cleaning up tmp")
os.system('rm -rf %s' % tmp_path)
log_message("Completed batch")
def main():
# time logging
global start_time
start_time = time.time()
import argparse
# parse commandline arguments
log_message(sys.argv)
parser = argparse.ArgumentParser(description='Generate synth dataset images.')
parser.add_argument('--idx', type=int,
help='idx of the requested sequence')
parser.add_argument('--ishape', type=int,
help='requested cut, according to the stride')
parser.add_argument('--stride', type=int,
help='stride amount, default 50')
args = parser.parse_args(sys.argv[sys.argv.index("--") + 1:])
idx = args.idx
ishape = args.ishape
stride = args.stride
log_message("input idx: %d" % idx)
log_message("input ishape: %d" % ishape)
log_message("input stride: %d" % stride)
if idx == None:
exit(1)
if ishape == None:
exit(1)
if stride == None:
log_message("WARNING: stride not specified, using default value 50")
stride = 50
# import idx info (name, split)
idx_info = load(open("pkl/idx_info.pickle", 'rb'))
# get runpass
(runpass, idx) = divmod(idx, len(idx_info))
log_message("runpass: %d" % runpass)
log_message("output idx: %d" % idx)
idx_info = idx_info[idx]
log_message("sequence: %s" % idx_info['name'])
log_message("nb_frames: %f" % idx_info['nb_frames'])
log_message("use_split: %s" % idx_info['use_split'])
# import configuration
log_message("Importing configuration")
import config
params = config.load_file('config', 'SYNTH_DATA')
smpl_data_folder = params['smpl_data_folder']
smpl_data_filename = params['smpl_data_filename']
bg_path = params['bg_path']
resy = params['resy']
resx = params['resx']
clothing_option = params['clothing_option'] # grey, nongrey or all
tmp_path = params['tmp_path']
output_path = params['output_path']
output_types = params['output_types']
stepsize = params['stepsize']
clipsize = params['clipsize']
openexr_py2_path = params['openexr_py2_path']
# compute number of cuts
nb_ishape = max(1, int(np.ceil((idx_info['nb_frames'] - (clipsize - stride))/stride)))
log_message("Max ishape: %d" % (nb_ishape - 1))
if ishape == None:
exit(1)
assert(ishape < nb_ishape)
# name is set given idx
name = idx_info['name']
output_path = join(output_path, 'run%d' % runpass, name.replace(" ", ""))
params['output_path'] = output_path
tmp_path = join(tmp_path, 'run%d_%s_c%04d' % (runpass, name.replace(" ", ""), (ishape + 1)))
params['tmp_path'] = tmp_path
# check if already computed
# + clean up existing tmp folders if any
if exists(tmp_path) and tmp_path != "" and tmp_path != "/":
os.system('rm -rf %s' % tmp_path)
rgb_vid_filename = "%s_c%04d.mp4" % (join(output_path, name.replace(' ', '')), (ishape + 1))
#if os.path.isfile(rgb_vid_filename):
# log_message("ALREADY COMPUTED - existing: %s" % rgb_vid_filename)
# return 0
# create tmp directory
if not exists(tmp_path):
mkdir_safe(tmp_path)
# >> don't use random generator before this point <<
# initialize RNG with seeds from sequence id
import hashlib
s = "synth_data:%d:%d:%d" % (idx, runpass,ishape)
seed_number = int(hashlib.sha1(s.encode('utf-8')).hexdigest(), 16) % (10 ** 8)
log_message("GENERATED SEED %d from string '%s'" % (seed_number, s))
random.seed(seed_number)
np.random.seed(seed_number)
if(output_types['vblur']):
vblur_factor = np.random.normal(0.5, 0.5)
params['vblur_factor'] = vblur_factor
log_message("Setup Blender")
# create copy-spher.harm. directory if not exists
sh_dir = join(tmp_path, 'spher_harm')
if not exists(sh_dir):
mkdir_safe(sh_dir)
sh_dst = join(sh_dir, 'sh_%02d_%05d.osl' % (runpass, idx))
os.system('cp spher_harm/sh.osl %s' % sh_dst)
genders = {0: 'female', 1: 'male'}
# pick random gender
gender = choice(genders)
scene = bpy.data.scenes['Scene']
scene.render.engine = 'CYCLES'
bpy.data.materials['Material'].use_nodes = True
scene.cycles.shading_system = True
scene.use_nodes = True
log_message("Listing background images")
bg_names = join(bg_path, '%s_img.txt' % idx_info['use_split'])
nh_txt_paths = []
with open(bg_names) as f:
for line in f:
nh_txt_paths.append(join(bg_path, line))
# grab clothing names
log_message("clothing: %s" % clothing_option)
with open( join(smpl_data_folder, 'textures', '%s_%s.txt' % ( gender, idx_info['use_split'] ) ) ) as f:
txt_paths = f.read().splitlines()
# if using only one source of clothing
if clothing_option == 'nongrey':
txt_paths = [k for k in txt_paths if 'nongrey' in k]
elif clothing_option == 'grey':
txt_paths = [k for k in txt_paths if 'nongrey' not in k]
# random clothing texture
cloth_img_name = choice(txt_paths)
cloth_img_name = join(smpl_data_folder, cloth_img_name)
cloth_img = bpy.data.images.load(cloth_img_name)
# random background
bg_img_name = choice(nh_txt_paths)[:-1]
bg_img = bpy.data.images.load(bg_img_name)
log_message("Loading parts segmentation")
beta_stds = np.load(join(smpl_data_folder, ('%s_beta_stds.npy' % gender)))
log_message("Building materials tree")
mat_tree = bpy.data.materials['Material'].node_tree
create_sh_material(mat_tree, sh_dst, cloth_img)
res_paths = create_composite_nodes(scene.node_tree, params, img=bg_img, idx=idx)
log_message("Loading smpl data")
smpl_data = np.load(join(smpl_data_folder, smpl_data_filename))
log_message("Initializing scene")
camera_distance = np.random.normal(8.0, 1)
params['camera_distance'] = camera_distance
ob, obname, arm_ob, cam_ob = init_scene(scene, params, gender)
setState0()
ob.select = True
bpy.context.scene.objects.active = ob
segmented_materials = True #True: 0-24, False: expected to have 0-1 bg/fg
log_message("Creating materials segmentation")
# create material segmentation
if segmented_materials:
materials = create_segmentation(ob, params)
prob_dressed = {'leftLeg':.5, 'leftArm':.9, 'leftHandIndex1':.01,
'rightShoulder':.8, 'rightHand':.01, 'neck':.01,
'rightToeBase':.9, 'leftShoulder':.8, 'leftToeBase':.9,
'rightForeArm':.5, 'leftHand':.01, 'spine':.9,
'leftFoot':.9, 'leftUpLeg':.9, 'rightUpLeg':.9,
'rightFoot':.9, 'head':.01, 'leftForeArm':.5,
'rightArm':.5, 'spine1':.9, 'hips':.9,
'rightHandIndex1':.01, 'spine2':.9, 'rightLeg':.5}
else:
materials = {'FullBody': bpy.data.materials['Material']}
prob_dressed = {'FullBody': .6}
orig_pelvis_loc = (arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy()) - Vector((-1., 1., 1.))
orig_cam_loc = cam_ob.location.copy()
# unblocking both the pose and the blendshape limits
for k in ob.data.shape_keys.key_blocks.keys():
bpy.data.shape_keys["Key"].key_blocks[k].slider_min = -10
bpy.data.shape_keys["Key"].key_blocks[k].slider_max = 10
log_message("Loading body data")
cmu_parms, fshapes, name = load_body_data(smpl_data, ob, obname, idx=idx, gender=gender)
log_message("Loaded body data for %s" % name)
nb_fshapes = len(fshapes)
if idx_info['use_split'] == 'train':
fshapes = fshapes[:int(nb_fshapes*0.8)]
elif idx_info['use_split'] == 'test':
fshapes = fshapes[int(nb_fshapes*0.8):]
# pick random real body shape
shape = choice(fshapes) #+random_shape(.5) can add noise
#shape = random_shape(3.) # random body shape
# example shapes
#shape = np.zeros(10) #average
#shape = np.array([ 2.25176191, -3.7883464 , 0.46747496, 3.89178988, 2.20098416, 0.26102114, -3.07428093, 0.55708514, -3.94442258, -2.88552087]) #fat
#shape = np.array([-2.26781107, 0.88158132, -0.93788176, -0.23480508, 1.17088298, 1.55550789, 0.44383225, 0.37688275, -0.27983086, 1.77102953]) #thin
#shape = np.array([ 0.00404852, 0.8084637 , 0.32332591, -1.33163664, 1.05008727, 1.60955275, 0.22372946, -0.10738459, 0.89456312, -1.22231216]) #short
#shape = np.array([ 3.63453289, 1.20836171, 3.15674431, -0.78646793, -1.93847355, -0.32129994, -0.97771656, 0.94531640, 0.52825811, -0.99324327]) #tall
ndofs = 10
scene.objects.active = arm_ob
orig_trans = np.asarray(arm_ob.pose.bones[obname+'_Pelvis'].location).copy()
# create output directory
if not exists(output_path):
mkdir_safe(output_path)
# spherical harmonics material needs a script to be loaded and compiled
scs = []
for mname, material in materials.items():
scs.append(material.node_tree.nodes['Script'])
scs[-1].filepath = sh_dst
scs[-1].update()
rgb_dirname = name.replace(" ", "") + '_c%04d.mp4' % (ishape + 1)
rgb_path = join(tmp_path, rgb_dirname)
data = cmu_parms[name]
fbegin = ishape*stepsize*stride
fend = min(ishape*stepsize*stride + stepsize*clipsize, len(data['poses']))
log_message("Computing how many frames to allocate")
N = len(data['poses'][fbegin:fend:stepsize])
log_message("Allocating %d frames in mat file" % N)
# force recomputation of joint angles unless shape is all zeros
curr_shape = np.zeros_like(shape)
nframes = len(data['poses'][::stepsize])
matfile_info = join(output_path, name.replace(" ", "") + "_c%04d_info.mat" % (ishape+1))
log_message('Working on %s' % matfile_info)
# allocate
dict_info = {}
dict_info['bg'] = np.zeros((N,), dtype=np.object) # background image path
dict_info['camLoc'] = np.empty(3) # (1, 3)
dict_info['clipNo'] = ishape +1
dict_info['cloth'] = np.zeros((N,), dtype=np.object) # clothing texture image path
dict_info['gender'] = np.empty(N, dtype='uint8') # 0 for male, 1 for female
dict_info['joints2D'] = np.empty((2, 24, N), dtype='float32') # 2D joint positions in pixel space
dict_info['joints3D'] = np.empty((3, 24, N), dtype='float32') # 3D joint positions in world coordinates
dict_info['light'] = np.empty((9, N), dtype='float32')
dict_info['pose'] = np.empty((data['poses'][0].size, N), dtype='float32') # joint angles from SMPL (CMU)
dict_info['sequence'] = name.replace(" ", "") + "_c%04d" % (ishape + 1)
dict_info['shape'] = np.empty((ndofs, N), dtype='float32')
dict_info['zrot'] = np.empty(N, dtype='float32')
dict_info['camDist'] = camera_distance
dict_info['stride'] = stride
if name.replace(" ", "").startswith('h36m'):
dict_info['source'] = 'h36m'
else:
dict_info['source'] = 'cmu'
if(output_types['vblur']):
dict_info['vblur_factor'] = np.empty(N, dtype='float32')
# for each clipsize'th frame in the sequence
get_real_frame = lambda ifr: ifr
random_zrot = 0
reset_loc = False
batch_it = 0
curr_shape = reset_joint_positions(orig_trans, shape, ob, arm_ob, obname, scene,
cam_ob, smpl_data['regression_verts'], smpl_data['joint_regressor'])
random_zrot = 2*np.pi*np.random.rand()
arm_ob.animation_data_clear()
cam_ob.animation_data_clear()
# create a keyframe animation with pose, translation, blendshapes and camera motion
# LOOP TO CREATE 3D ANIMATION
for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
iframe = seq_frame
scene.frame_set(get_real_frame(seq_frame))
# apply the translation, pose and shape to the character
apply_trans_pose_shape(Vector(trans), pose, shape, ob, arm_ob, obname, scene, cam_ob, get_real_frame(seq_frame))
dict_info['shape'][:, iframe] = shape[:ndofs]
dict_info['pose'][:, iframe] = pose
dict_info['gender'][iframe] = list(genders)[list(genders.values()).index(gender)]
if(output_types['vblur']):
dict_info['vblur_factor'][iframe] = vblur_factor
arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion(Euler((0, 0, random_zrot), 'XYZ'))
arm_ob.pose.bones[obname+'_root'].keyframe_insert('rotation_quaternion', frame=get_real_frame(seq_frame))
dict_info['zrot'][iframe] = random_zrot
scene.update()
# Bodies centered only in each minibatch of clipsize frames
if seq_frame == 0 or reset_loc:
reset_loc = False
new_pelvis_loc = arm_ob.matrix_world.copy() * arm_ob.pose.bones[obname+'_Pelvis'].head.copy()
cam_ob.location = orig_cam_loc.copy() + (new_pelvis_loc.copy() - orig_pelvis_loc.copy())
cam_ob.keyframe_insert('location', frame=get_real_frame(seq_frame))
dict_info['camLoc'] = np.array(cam_ob.location)
scene.node_tree.nodes['Image'].image = bg_img
for part, material in materials.items():
material.node_tree.nodes['Vector Math'].inputs[1].default_value[:2] = (0, 0)
# random light
sh_coeffs = .7 * (2 * np.random.rand(9) - 1)
sh_coeffs[0] = .5 + .9 * np.random.rand() # Ambient light (first coeff) needs a minimum is ambient. Rest is uniformly distributed, higher means brighter.
sh_coeffs[1] = -.7 * np.random.rand()
for ish, coeff in enumerate(sh_coeffs):
for sc in scs:
sc.inputs[ish+1].default_value = coeff
# iterate over the keyframes and render
# LOOP TO RENDER
for seq_frame, (pose, trans) in enumerate(zip(data['poses'][fbegin:fend:stepsize], data['trans'][fbegin:fend:stepsize])):
scene.frame_set(get_real_frame(seq_frame))
iframe = seq_frame
dict_info['bg'][iframe] = bg_img_name
dict_info['cloth'][iframe] = cloth_img_name
dict_info['light'][:, iframe] = sh_coeffs
scene.render.use_antialiasing = False
scene.render.filepath = join(rgb_path, 'Image%04d.png' % get_real_frame(seq_frame))
log_message("Rendering frame %d" % seq_frame)
# disable render output
logfile = '/dev/null'
open(logfile, 'a').close()
old = os.dup(1)
sys.stdout.flush()
os.close(1)
os.open(logfile, os.O_WRONLY)
# Render
bpy.ops.render.render(write_still=True)
# disable output redirection
os.close(1)
os.dup(old)
os.close(old)
# NOTE:
# ideally, pixels should be readable from a viewer node, but I get only zeros
# --> https://ammous88.wordpress.com/2015/01/16/blender-access-render-results-pixels-directly-from-python-2/
# len(np.asarray(bpy.data.images['Render Result'].pixels) is 0
# Therefore we write them to temporary files and read with OpenEXR library (available for python2) in main_part2.py
# Alternatively, if you don't want to use OpenEXR library, the following commented code does loading with Blender functions, but it can cause memory leak.
# If you want to use it, copy necessary lines from main_part2.py such as definitions of dict_normal, matfile_normal...
#for k, folder in res_paths.items():
# if not k== 'vblur' and not k=='fg':
# path = join(folder, 'Image%04d.exr' % get_real_frame(seq_frame))
# render_img = bpy.data.images.load(path)
# # render_img.pixels size is width * height * 4 (rgba)
# arr = np.array(render_img.pixels[:]).reshape(resx, resy, 4)[::-1,:, :] # images are vertically flipped
# if k == 'normal':# 3 channels, original order
# mat = arr[:,:, :3]
# dict_normal['normal_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'gtflow':
# mat = arr[:,:, 1:3]
# dict_gtflow['gtflow_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'depth':
# mat = arr[:,:, 0]
# dict_depth['depth_%d' % (iframe + 1)] = mat.astype(np.float32, copy=False)
# elif k == 'segm':
# mat = arr[:,:,0]
# dict_segm['segm_%d' % (iframe + 1)] = mat.astype(np.uint8, copy=False)
#
# # remove the image to release memory, object handles, etc.
# render_img.user_clear()
# bpy.data.images.remove(render_img)
# bone locations should be saved after rendering so that the bones are updated
bone_locs_2D, bone_locs_3D = get_bone_locs(obname, arm_ob, scene, cam_ob)
dict_info['joints2D'][:, :, iframe] = np.transpose(bone_locs_2D)
dict_info['joints3D'][:, :, iframe] = np.transpose(bone_locs_3D)
reset_loc = (bone_locs_2D.max(axis=-1) > 256).any() or (bone_locs_2D.min(axis=0) < 0).any()
arm_ob.pose.bones[obname+'_root'].rotation_quaternion = Quaternion((1, 0, 0, 0))
# save a .blend file for debugging:
# bpy.ops.wm.save_as_mainfile(filepath=join(tmp_path, 'pre.blend'))
# save RGB data with ffmpeg (if you don't have h264 codec, you can replace with another one and control the quality with something like -q:v 3)
cmd_ffmpeg = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 ''%s_c%04d.mp4''' % (join(rgb_path, 'Image%04d.png'), join(output_path, name.replace(' ', '')), (ishape + 1))
log_message("Generating RGB video (%s)" % cmd_ffmpeg)
os.system(cmd_ffmpeg)
if(output_types['vblur']):
cmd_ffmpeg_vblur = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 -vf "scale=trunc(iw/2)*2:trunc(ih/2)*2" ''%s_c%04d.mp4''' % (join(res_paths['vblur'], 'Image%04d.png'), join(output_path, name.replace(' ', '')+'_vblur'), (ishape + 1))
log_message("Generating vblur video (%s)" % cmd_ffmpeg_vblur)
os.system(cmd_ffmpeg_vblur)
if(output_types['fg']):
cmd_ffmpeg_fg = 'ffmpeg -y -r 30 -i ''%s'' -c:v h264 -pix_fmt yuv420p -crf 23 ''%s_c%04d.mp4''' % (join(res_paths['fg'], 'Image%04d.png'), join(output_path, name.replace(' ', '')+'_fg'), (ishape + 1))
log_message("Generating fg video (%s)" % cmd_ffmpeg_fg)
os.system(cmd_ffmpeg_fg)
cmd_tar = 'tar -czvf %s/%s.tar.gz -C %s %s' % (output_path, rgb_dirname, tmp_path, rgb_dirname)
log_message("Tarballing the images (%s)" % cmd_tar)
os.system(cmd_tar)
# save annotation excluding png/exr data to _info.mat file
import scipy.io
scipy.io.savemat(matfile_info, dict_info, do_compression=True)
