def run_face_expressivity(video_uri, out_dir, f_cfg): """ Processing all patient's for fetching facial landmarks --------------- --------------- Args: video_uri: video path; f_cfg: raw variable config object out_dir: (str) Output directory for processed output """ try: #Baseline logic cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) if len(of_csv_path) > 0: df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) df_of = df_of[cfr.AU_fl] expr_df_list = of_feature(df_of, cfr, f_cfg) exp_final_df = pd.concat(expr_df_list, ignore_index=True) exp_final_df['dbm_master_url'] = video_uri logger.info('Processing Output file {} '.format( os.path.join(out_loc, fl_name))) ut.save_output(exp_final_df, out_loc, fl_name, face_expr_dir, csv_ext) except Exception as e: logger.error('Failed to process video file')
def calc_gne(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing gne matrix Args: audio_file: (.wav) parsed audio file out_loc: (str) Output directory for csv's """ dir_path = os.path.join(out_loc, ff_dir) if os.path.isdir(dir_path): voice_seg = segment_pitch(dir_path, r_config) gne_all_frames = [np.NaN] * len(voice_seg[0]) gne_segment_frames = segment_gne(voice_seg[0], voice_seg[1], voice_seg[2], gne_all_frames, audio_file) df_gne = pd.DataFrame(gne_segment_frames, columns=[r_config.aco_gne]) df_gne[ r_config. err_reason] = 'Pass' # will replace with threshold in future release df_gne['Frames'] = df_gne.index df_gne['dbm_master_url'] = video_uri logger.info('Processing Output file {} '.format(out_loc)) ut.save_output(df_gne, out_loc, fl_name, gne_dir, csv_ext) else: error_txt = 'error: pitch freq not available' empty_gne(video_uri, out_loc, fl_name, r_config, error_txt)
def run_face_asymmetry(video_uri, out_dir, f_cfg): """ Processing all patient's for calculating facial asymmetry --------------- --------------- Args: video_uri: video path; f_cfg: face config object out_dir: (str) Output directory for processed output """ try: #Baseline logic cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) if len(of_csv_path) > 0: of_csv = of_csv_path[0] asym_df_list = calc_asym_feature(of_csv, f_cfg) asym_final_df = pd.concat(asym_df_list, ignore_index=True) asym_final_df['dbm_master_url'] = video_uri logger.info('Processing Output file {} '.format( os.path.join(out_loc, fl_name))) ut.save_output(asym_final_df, out_loc, fl_name, face_asym_dir, csv_ext) except Exception as e: logger.error('Failed to process video file')
def calc_shimmer(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing shimmer matrix Args: audio_file: (.wav) parsed audio file out_loc: (str) Output directory for csv r_config: config.config_raw_feature.pyConfigFeatureNmReader object """ dir_path = os.path.join(out_loc, ff_dir) if os.path.isdir(dir_path): voice_seg = segment_pitch(dir_path, r_config) shimmer_frames = [np.NaN] * len(voice_seg[0]) shimmer_segment_frames = segment_shimmer(voice_seg[0], voice_seg[1], voice_seg[2], shimmer_frames, audio_file) df_shimmer = pd.DataFrame(shimmer_segment_frames, columns=[r_config.aco_shimmer]) df_shimmer[ r_config. err_reason] = 'Pass' # will replace with threshold in future release df_shimmer['Frames'] = df_shimmer.index df_shimmer['dbm_master_url'] = video_uri logger.info('Processing Output file {} '.format(out_loc)) ut.save_output(df_shimmer, out_loc, fl_name, shimmer_dir, csv_ext) else: error_txt = 'error: fundamental freq not available' empty_shimmer(video_uri, out_loc, fl_name, r_config, error_txt)
def calc_head_mov(video_uri, df_of, out_loc, fl_name, r_config): """ Computing head motion and head pose variables Args: df_of: Openface dataframe out_loc: Output path for saving output csv's fl_name: file name for output csv r_config: raw variable config file object """ col = [ ' confidence', ' pose_Rx', ' pose_Ry', ' pose_Rz', ' pose_Tx', ' pose_Ty', ' pose_Tz' ] df_of = df_of[col] df_hmotion = head_vel(df_of, r_config) df_hmotion['dbm_master_url'] = video_uri df_pose = head_pose(df_of, r_config) df_pose['dbm_master_url'] = video_uri ut.save_output(df_hmotion, out_loc, fl_name, h_mov_dir, h_mov_ext) ut.save_output(df_pose, out_loc, fl_name, h_pose_dir, h_pose_ext)
def empty_intensity(video_uri, out_loc, fl_name, r_config): """ Preparing empty Intensity matrix if something fails """ cols = ['Frames', r_config.aco_int, r_config.err_reason] out_val = [[np.nan, np.nan, error_txt]] df_int = pd.DataFrame(out_val, columns = cols) df_int['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_int, out_loc, fl_name, intensity_dir, csv_ext)
def run_pause_segment(video_uri, out_dir, r_config): """ Processing all patient's for getting Pause Segment --------------- --------------- Args: video_uri: video path; r_config: raw variable config object out_dir: (str) Output directory for processed output """ try: input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) aud_filter = glob.glob(join(input_loc, fl_name + '.wav')) if len(aud_filter) > 0: audio_file = aud_filter[0] aud_dur = librosa.get_duration(filename=audio_file) if float(aud_dur) < 0.064: logger.info('Output file {} size is less than 0.064sec'.format( audio_file)) error_txt = 'error: length less than 0.064' empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt) return logger.info('Converting stereo sound to mono-lD') sound_mono = AudioSegment.from_wav(audio_file) sound_mono = sound_mono.set_channels(1) sound_mono = sound_mono.set_frame_rate(48000) mono_wav = os.path.join(input_loc, fl_name + '_mono.wav') sound_mono.export(mono_wav, format="wav") df_pause_seg = process_silence(mono_wav, r_config) os.remove(mono_wav) #removing mono wav file if isinstance(df_pause_seg, pd.DataFrame) and len(df_pause_seg) > 0: logger.info('Processing Output file {} '.format(out_loc)) df_pause_seg['dbm_master_url'] = video_uri ut.save_output(df_pause_seg, out_loc, fl_name, pause_seg_dir, csv_ext) else: error_txt = 'error: webrtcvad returns no segment' empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt) except Exception as e: logger.error('Failed to process audio file')
def empty_fm(video_uri, out_loc, fl_name, r_config): """ Preparing empty formant frequency matrix if something fails """ cols = ['Frames', r_config.aco_fm1, r_config.aco_fm2, r_config.aco_fm3, r_config.aco_fm4, r_config.err_reason] out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] df_fm = pd.DataFrame(out_val, columns = cols) df_fm['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_fm, out_loc, fl_name, formant_dir, csv_ext)
def empty_vfs(video_uri, out_loc, fl_name, r_config): """ Preparing empty VFS matrix if something fails """ cols = [ 'Frames', r_config.aco_voiceFrame, r_config.aco_totVoiceFrame, r_config.aco_voicePct, r_config.err_reason ] out_val = [[np.nan, np.nan, np.nan, np.nan, error_txt]] df_vfs = pd.DataFrame(out_val, columns=cols) df_vfs['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def empty_pause_segment(video_uri, out_loc, fl_name, r_config, error_txt): """ Preparing empty Pause Segment matrix if something fails """ cols = [ r_config.aco_totaltime, r_config.aco_speakingtime, r_config.aco_numpauses, r_config.aco_pausetime, r_config.aco_pausefrac, r_config.err_reason ] out_val = [[np.nan, np.nan, np.nan, np.nan, np.nan, error_txt]] df_pause = pd.DataFrame(out_val, columns=cols) df_pause['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_pause, out_loc, fl_name, pause_seg_dir, csv_ext)
def calc_intensity(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing Intensity matrix Args: audio_file: (.wav) parsed audio file out_loc: (str) Output directory for csv's """ intensity_frames = intensity_score(audio_file) df_intensity = pd.DataFrame(intensity_frames, columns=[r_config.aco_int]) df_intensity['Frames'] = df_intensity.index df_intensity['dbm_master_url'] = video_uri df_intensity[r_config.err_reason] = 'Pass'# will replace with threshold in future release logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_intensity, out_loc, fl_name, intensity_dir, csv_ext)
def calc_hnr(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing harmonic noise matrix Args: audio_file: (.wav) parsed audio file out_loc: (str) Output directory for csv's """ hnr_all_frames = hnr_ratio(audio_file) df_hnr = pd.DataFrame(hnr_all_frames, columns=[r_config.aco_hnr]) df_hnr['Frames'] = df_hnr.index df_hnr['dbm_master_url'] = video_uri df_hnr[ r_config. err_reason] = 'Pass' # will replace with threshold in future release logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_hnr, out_loc, fl_name, hnr_dir, csv_ext)
def empty_mfcc(video_uri, out_loc, fl_name, r_config): """ Preparing empty empty_mfcc matrix if something fails """ cols = [ 'Frames', r_config.aco_mfcc1, r_config.aco_mfcc2, r_config.aco_mfcc3, r_config.aco_mfcc4, r_config.aco_mfcc5, r_config.aco_mfcc6, r_config.aco_mfcc7, r_config.aco_mfcc8, r_config.aco_mfcc9, r_config.aco_mfcc10, r_config.aco_mfcc11, r_config.aco_mfcc12, r_config.err_reason ] out_val = [[ np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, error_txt ]] df_mfcc = pd.DataFrame(out_val, columns=cols) df_mfcc['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_mfcc, out_loc, fl_name, mfcc_dir, csv_ext)
def calc_transcribe(video_uri, audio_file, out_loc, fl_name, r_config, deep_path, aud_dur): """ Preparing Formant freq matrix Args: audio_file: (.wav) parsed audio file; fl_name: input file name out_loc: (str) Output directory; r_config: raw variable config """ text = n_util.process_deepspeech(audio_file, deep_path) df_formant = pd.DataFrame([text], columns=[r_config.nlp_transcribe]) df_formant.replace('', np.nan, regex=True, inplace=True) df_formant[r_config.nlp_totalTime] = aud_dur df_formant[ r_config. err_reason] = 'Pass' # will replace with threshold in future release df_formant['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def calc_eye_mov(video_uri, df_of, out_loc, fl_name, r_config): """ Computing eye motion variables Args: df_of: Openface dataframe out_loc: Output path for saving output csv's fl_name: file name for output csv r_config: raw variable config file object """ col_l = [' gaze_0_x', ' gaze_0_y', ' gaze_0_z'] col_r = [' gaze_1_x', ' gaze_1_y', ' gaze_1_z'] gazel_disp, err_l = eye_disp(df_of, col_l, r_config) gazer_disp, err_r = eye_disp(df_of, col_r, r_config) df_disp = eye_motion_df(gazel_disp, gazer_disp, err_l, r_config) df_disp['dbm_master_url'] = video_uri df_motion = filter_motion(df_of, df_disp, col_l, col_r, r_config) ut.save_output(df_motion, out_loc, fl_name, eye_pose_dir, eye_pose_ext)
def run_face_landmark(video_uri, out_dir, f_cfg): """ Processing all patient's for fetching facial landmarks --------------- --------------- Args: video_uri: video path; f_cfg: raw variable config object out_dir: (str) Output directory for processed output """ try: #Baseline logic cfr = ConfigFaceReader() input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) of_csv_path = glob.glob(join(out_loc, fl_name + '_OF_features/*.csv')) if len(of_csv_path) > 0: df_of = pd.read_csv(of_csv_path[0], error_bad_lines=False) df_lmk = df_of[extract_col_nm_lmk(df_of)] df_lmk = df_lmk.copy() df_lmk['frame'] = df_of['frame'] df_lmk['face_id'] = df_of[' face_id'] df_lmk['timestamp'] = df_of[' timestamp'] df_lmk['confidence'] = df_of[' confidence'] df_lmk['success'] = df_of[' success'] df_lmk = lmk_col_nm_map(df_lmk) df_lmk = add_disp_3D(df_lmk) df_lmk['dbm_master_url'] = video_uri logger.info('Processing Output file {} '.format( join(out_loc, fl_name))) ut.save_output(df_lmk, out_loc, fl_name, face_lmk_dir, csv_ext) except Exception as e: logger.error('Failed to process video file')
def calc_vfs(video_uri, audio_file, out_loc, fl_name, r_config): """ creating dataframe matrix for voice frame score Args: audio_file: Audio file path new_out_base_dir: AWS instance output base directory path f_nm_config: Config file object """ voice_percentage, voiced_frames, total_frames = voice_segment(audio_file) df_vfs = pd.DataFrame([voiced_frames], columns=[r_config.aco_voiceFrame]) df_vfs[r_config.aco_totVoiceFrame] = [total_frames] df_vfs[r_config.aco_voicePct] = [voice_percentage] df_vfs[ r_config. err_reason] = 'Pass' # will replace with threshold in future release df_vfs['Frames'] = df_vfs.index df_vfs['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_vfs, out_loc, fl_name, vfs_dir, csv_ext)
def calc_formant(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing Formant freq matrix Args: audio_file: (.wav) parsed audio file; fl_name: input file name out_loc: (str) Output directory; r_config: raw variable config """ f1_list,f2_list,f3_list,f4_list = formant_score(audio_file) df_formant = pd.DataFrame(f1_list, columns=[r_config.aco_fm1]) df_formant[r_config.aco_fm2] = f2_list df_formant[r_config.aco_fm3] = f3_list df_formant[r_config.aco_fm4] = f4_list df_formant.replace('', np.nan, regex=True,inplace=True) df_formant[r_config.err_reason] = 'Pass'# will replace with threshold in future release df_formant['Frames'] = df_formant.index df_formant['dbm_master_url'] = video_uri logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_formant, out_loc, fl_name, formant_dir, csv_ext)
def run_speech_feature(video_uri, out_dir, r_config): """ Processing all patient's for fetching nlp features ------------------- ------------------- Args: video_uri: video path; r_config: raw variable config object out_dir: (str) Output directory for processed output """ try: input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) transcribe_path = glob.glob(join(out_loc, transcribe_ext)) if len(transcribe_path)>0: transcribe_df = pd.read_csv(transcribe_path[0]) df_speech= n_util.process_speech(transcribe_df, r_config) logger.info('Saving Output file {} '.format(out_loc)) ut.save_output(df_speech, out_loc, fl_name, speech_dir, speech_ext) except Exception as e: logger.error('Failed to process video file')
def calc_mfcc(video_uri, audio_file, out_loc, fl_name, r_config): """ Preparing mfcc matrix Args: audio_file: (.wav) parsed audio file out_loc: output location to save csv fl_name: (str) name of audio file r_config: config.config_raw_feature.pyConfigFeatureNmReader object """ dict_ = {} mfccs = audio_mfcc(audio_file) for i in range(1, 13): conf_str = r_config.base_raw['raw_feature'] dict_[conf_str['aco_mfcc' + str(i)]] = mfccs[i - 1, :] df = pd.DataFrame(dict_) df['Frames'] = df.index df[r_config. err_reason] = 'Pass' # may replace based on threshold in future release df['dbm_master_url'] = video_uri ut.save_output(df, out_loc, fl_name, mfcc_dir, csv_ext)
def run_eye_blink(video_uri, out_dir, r_config, facial_landmarks): """ Processing all patient's for getting eye blink artifacts --------------- --------------- Args: video_uri: video path; input_dir : input directory for video's out_dir: (str) Output directory for processed output; r_config: raw variable config object; facial_landmarks: landmark model path """ try: input_loc, out_loc, fl_name = ut.filter_path(video_uri, out_dir) vid_file_path = os.path.exists(video_uri) if vid_file_path == True: logger.info('Processing Output file {} '.format( os.path.join(out_loc, fl_name))) df_blink = blink_detection(video_uri, facial_landmarks, r_config) ut.save_output(df_blink, out_loc, fl_name, movement_expr_dir, csv_ext) except Exception as e: logger.error('Failed to process video file')