def run_face_expressivity(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
try:
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path) > 0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_of = df_of[cfr.AU_fl]
expr_df_list = of_feature(df_of, cfr, f_cfg)
exp_final_df = pd.concat(expr_df_list, ignore_index=True)
exp_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(
os.path.join(out_loc, fl_name)))
ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir,
csv_ext)
except Exception as e:
logger.error('Failed to process video file')
def calc_gne(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing gne matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
dir_path = os.path.join(out_loc, ff_dir)
if os.path.isdir(dir_path):
voice_seg = segment_pitch(dir_path, r_config)
gne_all_frames = [np.NaN] * len(voice_seg[0])
gne_segment_frames = segment_gne(voice_seg[0], voice_seg[1],
voice_seg[2], gne_all_frames,
audio_file)
df_gne = pd.DataFrame(gne_segment_frames, columns=[r_config.aco_gne])
df_gne[
r_config.
err_reason] = 'Pass' # will replace with threshold in future release
df_gne['Frames'] = df_gne.index
df_gne['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext)
else:
error_txt = 'error: pitch freq not available'
empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
def run_face_asymmetry(video_uri, out_dir, f_cfg):
"""
Processing all patient's for calculating facial asymmetry
---------------
---------------
Args:
video_uri: video path; f_cfg: face config object
out_dir: (str) Output directory for processed output
"""
try:
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path) > 0:
of_csv = of_csv_path[0]
asym_df_list = calc_asym_feature(of_csv, f_cfg)
asym_final_df = pd.concat(asym_df_list, ignore_index=True)
asym_final_df['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(
os.path.join(out_loc, fl_name)))
ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir,
csv_ext)
except Exception as e:
logger.error('Failed to process video file')
def calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing shimmer matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv
r_config: config.config_raw_feature.pyConfigFeatureNmReader object
"""
dir_path = os.path.join(out_loc, ff_dir)
if os.path.isdir(dir_path):
voice_seg = segment_pitch(dir_path, r_config)
shimmer_frames = [np.NaN] * len(voice_seg[0])
shimmer_segment_frames = segment_shimmer(voice_seg[0], voice_seg[1],
voice_seg[2], shimmer_frames,
audio_file)
df_shimmer = pd.DataFrame(shimmer_segment_frames,
columns=[r_config.aco_shimmer])
df_shimmer[
r_config.
err_reason] = 'Pass' # will replace with threshold in future release
df_shimmer['Frames'] = df_shimmer.index
df_shimmer['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(out_loc))
ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext)
else:
error_txt = 'error: fundamental freq not available'
empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
def calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config):
"""
Computing head motion and head pose variables
Args:
df_of: Openface dataframe
out_loc: Output path for saving output csv's
fl_name: file name for output csv
r_config: raw variable config file object
"""
col = [
' confidence', ' pose_Rx', ' pose_Ry', ' pose_Rz', ' pose_Tx',
' pose_Ty', ' pose_Tz'
]
df_of = df_of[col]
df_hmotion = head_vel(df_of, r_config)
df_hmotion['dbm_master_url'] = video_uri
df_pose = head_pose(df_of, r_config)
df_pose['dbm_master_url'] = video_uri
ut.save_output(df_hmotion, out_loc, fl_name, h_mov_dir, h_mov_ext)
ut.save_output(df_pose, out_loc, fl_name, h_pose_dir, h_pose_ext)
def empty_intensity(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty Intensity matrix if something fails
"""
cols = ['Frames', r_config.aco_int, r_config.err_reason]
out_val = [[np.nan, np.nan, error_txt]]
df_int = pd.DataFrame(out_val, columns = cols)
df_int['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_int, out_loc, fl_name, intensity_dir, csv_ext)
def run_pause_segment(video_uri, out_dir, r_config):
"""
Processing all patient's for getting Pause Segment
---------------
---------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
aud_filter = glob.glob(join(input_loc, fl_name + '.wav'))
if len(aud_filter) > 0:
audio_file = aud_filter[0]
aud_dur = librosa.get_duration(filename=audio_file)
if float(aud_dur) < 0.064:
logger.info('Output file {} size is less than 0.064sec'.format(
audio_file))
error_txt = 'error: length less than 0.064'
empty_pause_segment(video_uri, out_loc, fl_name, r_config,
error_txt)
return
logger.info('Converting stereo sound to mono-lD')
sound_mono = AudioSegment.from_wav(audio_file)
sound_mono = sound_mono.set_channels(1)
sound_mono = sound_mono.set_frame_rate(48000)
mono_wav = os.path.join(input_loc, fl_name + '_mono.wav')
sound_mono.export(mono_wav, format="wav")
df_pause_seg = process_silence(mono_wav, r_config)
os.remove(mono_wav) #removing mono wav file
if isinstance(df_pause_seg,
pd.DataFrame) and len(df_pause_seg) > 0:
logger.info('Processing Output file {} '.format(out_loc))
df_pause_seg['dbm_master_url'] = video_uri
ut.save_output(df_pause_seg, out_loc, fl_name, pause_seg_dir,
csv_ext)
else:
error_txt = 'error: webrtcvad returns no segment'
empty_pause_segment(video_uri, out_loc, fl_name, r_config,
error_txt)
except Exception as e:
logger.error('Failed to process audio file')
def empty_fm(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty formant frequency matrix if something fails
"""
cols = ['Frames', r_config.aco_fm1, r_config.aco_fm2, r_config.aco_fm3, r_config.aco_fm4, r_config.err_reason]
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
df_fm = pd.DataFrame(out_val, columns = cols)
df_fm['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
def empty_vfs(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty VFS matrix if something fails
"""
cols = [
'Frames', r_config.aco_voiceFrame, r_config.aco_totVoiceFrame,
r_config.aco_voicePct, r_config.err_reason
]
out_val = [[np.nan, np.nan, np.nan, np.nan, error_txt]]
df_vfs = pd.DataFrame(out_val, columns=cols)
df_vfs['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt):
"""
Preparing empty Pause Segment matrix if something fails
"""
cols = [
r_config.aco_totaltime, r_config.aco_speakingtime,
r_config.aco_numpauses, r_config.aco_pausetime, r_config.aco_pausefrac,
r_config.err_reason
]
out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]]
df_pause = pd.DataFrame(out_val, columns=cols)
df_pause['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_pause, out_loc, fl_name, pause_seg_dir, csv_ext)
def calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing Intensity matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
intensity_frames = intensity_score(audio_file)
df_intensity = pd.DataFrame(intensity_frames, columns=[r_config.aco_int])
df_intensity['Frames'] = df_intensity.index
df_intensity['dbm_master_url'] = video_uri
df_intensity[r_config.err_reason] = 'Pass'# will replace with threshold in future release
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_intensity, out_loc, fl_name, intensity_dir, csv_ext)
def calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing harmonic noise matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: (str) Output directory for csv's
"""
hnr_all_frames = hnr_ratio(audio_file)
df_hnr = pd.DataFrame(hnr_all_frames, columns=[r_config.aco_hnr])
df_hnr['Frames'] = df_hnr.index
df_hnr['dbm_master_url'] = video_uri
df_hnr[
r_config.
err_reason] = 'Pass' # will replace with threshold in future release
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
def empty_mfcc(video_uri, out_loc, fl_name, r_config):
"""
Preparing empty empty_mfcc matrix if something fails
"""
cols = [
'Frames', r_config.aco_mfcc1, r_config.aco_mfcc2, r_config.aco_mfcc3,
r_config.aco_mfcc4, r_config.aco_mfcc5, r_config.aco_mfcc6,
r_config.aco_mfcc7, r_config.aco_mfcc8, r_config.aco_mfcc9,
r_config.aco_mfcc10, r_config.aco_mfcc11, r_config.aco_mfcc12,
r_config.err_reason
]
out_val = [[
np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan,
np.nan, np.nan, np.nan, np.nan, error_txt
]]
df_mfcc = pd.DataFrame(out_val, columns=cols)
df_mfcc['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_mfcc, out_loc, fl_name, mfcc_dir, csv_ext)
def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config,
deep_path, aud_dur):
"""
Preparing Formant freq matrix
Args:
audio_file: (.wav) parsed audio file; fl_name: input file name
out_loc: (str) Output directory; r_config: raw variable config
"""
text = n_util.process_deepspeech(audio_file, deep_path)
df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe])
df_formant.replace('', np.nan, regex=True, inplace=True)
df_formant[r_config.nlp_totalTime] = aud_dur
df_formant[
r_config.
err_reason] = 'Pass' # will replace with threshold in future release
df_formant['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config):
"""
Computing eye motion variables
Args:
df_of: Openface dataframe
out_loc: Output path for saving output csv's
fl_name: file name for output csv
r_config: raw variable config file object
"""
col_l = [' gaze_0_x', ' gaze_0_y', ' gaze_0_z']
col_r = [' gaze_1_x', ' gaze_1_y', ' gaze_1_z']
gazel_disp, err_l = eye_disp(df_of, col_l, r_config)
gazer_disp, err_r = eye_disp(df_of, col_r, r_config)
df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config)
df_disp['dbm_master_url'] = video_uri
df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config)
ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
def run_face_landmark(video_uri, out_dir, f_cfg):
"""
Processing all patient's for fetching facial landmarks
---------------
---------------
Args:
video_uri: video path; f_cfg: raw variable config object
out_dir: (str) Output directory for processed output
"""
try:
#Baseline logic
cfr = ConfigFaceReader()
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv'))
if len(of_csv_path) > 0:
df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False)
df_lmk = df_of[extract_col_nm_lmk(df_of)]
df_lmk = df_lmk.copy()
df_lmk['frame'] = df_of['frame']
df_lmk['face_id'] = df_of[' face_id']
df_lmk['timestamp'] = df_of[' timestamp']
df_lmk['confidence'] = df_of[' confidence']
df_lmk['success'] = df_of[' success']
df_lmk = lmk_col_nm_map(df_lmk)
df_lmk = add_disp_3D(df_lmk)
df_lmk['dbm_master_url'] = video_uri
logger.info('Processing Output file {} '.format(
join(out_loc, fl_name)))
ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext)
except Exception as e:
logger.error('Failed to process video file')
def calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config):
"""
creating dataframe matrix for voice frame score
Args:
audio_file: Audio file path
new_out_base_dir: AWS instance output base directory path
f_nm_config: Config file object
"""
voice_percentage, voiced_frames, total_frames = voice_segment(audio_file)
df_vfs = pd.DataFrame([voiced_frames], columns=[r_config.aco_voiceFrame])
df_vfs[r_config.aco_totVoiceFrame] = [total_frames]
df_vfs[r_config.aco_voicePct] = [voice_percentage]
df_vfs[
r_config.
err_reason] = 'Pass' # will replace with threshold in future release
df_vfs['Frames'] = df_vfs.index
df_vfs['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def calc_formant(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing Formant freq matrix
Args:
audio_file: (.wav) parsed audio file; fl_name: input file name
out_loc: (str) Output directory; r_config: raw variable config
"""
f1_list,f2_list,f3_list,f4_list = formant_score(audio_file)
df_formant = pd.DataFrame(f1_list, columns=[r_config.aco_fm1])
df_formant[r_config.aco_fm2] = f2_list
df_formant[r_config.aco_fm3] = f3_list
df_formant[r_config.aco_fm4] = f4_list
df_formant.replace('', np.nan, regex=True,inplace=True)
df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release
df_formant['Frames'] = df_formant.index
df_formant['dbm_master_url'] = video_uri
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def run_speech_feature(video_uri, out_dir, r_config):
"""
Processing all patient's for fetching nlp features
-------------------
-------------------
Args:
video_uri: video path; r_config: raw variable config object
out_dir: (str) Output directory for processed output
"""
try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
transcribe_path = glob.glob(join(out_loc, transcribe_ext))
if len(transcribe_path)>0:
transcribe_df = pd.read_csv(transcribe_path[0])
df_speech= n_util.process_speech(transcribe_df, r_config)
logger.info('Saving Output file {} '.format(out_loc))
ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext)
except Exception as e:
logger.error('Failed to process video file')
def calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config):
"""
Preparing mfcc matrix
Args:
audio_file: (.wav) parsed audio file
out_loc: output location to save csv
fl_name: (str) name of audio file
r_config: config.config_raw_feature.pyConfigFeatureNmReader object
"""
dict_ = {}
mfccs = audio_mfcc(audio_file)
for i in range(1, 13):
conf_str = r_config.base_raw['raw_feature']
dict_[conf_str['aco_mfcc' + str(i)]] = mfccs[i - 1, :]
df = pd.DataFrame(dict_)
df['Frames'] = df.index
df[r_config.
err_reason] = 'Pass' # may replace based on threshold in future release
df['dbm_master_url'] = video_uri
ut.save_output(df, out_loc, fl_name, mfcc_dir, csv_ext)
def run_eye_blink(video_uri, out_dir, r_config, facial_landmarks):
"""
Processing all patient's for getting eye blink artifacts
---------------
---------------
Args:
video_uri: video path; input_dir : input directory for video's
out_dir: (str) Output directory for processed output; r_config: raw variable config object;
facial_landmarks: landmark model path
"""
try:
input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir)
vid_file_path = os.path.exists(video_uri)
if vid_file_path == True:
logger.info('Processing Output file {} '.format(
os.path.join(out_loc, fl_name)))
df_blink = blink_detection(video_uri, facial_landmarks, r_config)
ut.save_output(df_blink, out_loc, fl_name, movement_expr_dir,
csv_ext)
except Exception as e:
logger.error('Failed to process video file')
