def test_moment(self): y = mstats.moment(self.testcase,1) assert_almost_equal(y,0.0,10) y = mstats.moment(self.testcase,2) assert_almost_equal(y,1.25) y = mstats.moment(self.testcase,3) assert_almost_equal(y,0.0) y = mstats.moment(self.testcase,4) assert_almost_equal(y,2.5625)
def test_moment(self): y = mstats.moment(self.testcase,1) assert_almost_equal(y,0.0,10) y = mstats.moment(self.testcase,2) assert_almost_equal(y,1.25) y = mstats.moment(self.testcase,3) assert_almost_equal(y,0.0) y = mstats.moment(self.testcase,4) assert_almost_equal(y,2.5625)
def test_moment(self): # mean((testcase-mean(testcase))**power,axis=0),axis=0))**power)) y = mstats.moment(self.testcase,1) assert_almost_equal(y,0.0,10) y = mstats.moment(self.testcase,2) assert_almost_equal(y,1.25) y = mstats.moment(self.testcase,3) assert_almost_equal(y,0.0) y = mstats.moment(self.testcase,4) assert_almost_equal(y,2.5625)
def test_moment(self): # mean((testcase-mean(testcase))**power,axis=0),axis=0))**power)) y = mstats.moment(self.testcase,1) assert_almost_equal(y,0.0,10) y = mstats.moment(self.testcase,2) assert_almost_equal(y,1.25) y = mstats.moment(self.testcase,3) assert_almost_equal(y,0.0) y = mstats.moment(self.testcase,4) assert_almost_equal(y,2.5625)
def calc_moments(self, data, ex_dict): """ Calculate four moments of one exercise data : input matrix and the dimension is rows number x 3 ex_dict : resistance exercises dictionary return : feature matrix 4x3 where, 4 : moments; 3 : x, y, z """ m1 = np.mean(data, axis=0) m2 = np.var(data, axis=0) m2 = np.power(m2, 1.0 / 2) m3 = moment(data, 3, axis=0) m3 = np.power(abs(m3), 1.0 / 3) * np.sign(m3) m4 = moment(data, 4, axis=0) m4 = np.power(abs(m4), 1.0 / 4) ft_mat = np.concatenate((m1, m2, m3, m4), axis=0) ft_mat = ft_mat.reshape(self.moments_num, self.axises_num) return ft_mat
def q2a(function, runs): samplesHolder = [] posnHolder = [] rmsSeries = [] for i in range(0, runs): x, gamma, eigval, eigvect, lambdaMatrix = loop(function,10,0.1) samplesHolder.append(x) plt.plot(x,color='grey', alpha=0.5) plt.plot(np.mean(samplesHolder, axis=0),linewidth=2, color = 'black') plt.plot(moment(samplesHolder, moment = 2, axis=0), linewidth=2, color = 'blue') plt.plot(moment(samplesHolder, moment = 3, axis=0), linewidth=2, color = 'green') plt.plot(moment(samplesHolder, moment = 4, axis=0), linewidth=2, color = 'red') for sample in samplesHolder: posList = [] for i in range(0,len(sample)): posList.append(sum(sample[0:i])) posnHolder.append(posList) return posnHolder
def score(self,x,y): # x_dm = self.dm.transform(x) # test_dist = sklearn.metrics.pairwise_distances(self.dm.data_dm().x,x_dm) # test_min_dist = numpy.min(test_dist,axis=0) # closer = test_min_dist < self.max_distance weight, closer = self.weight(x) weight_sort=numpy.sort(weight[closer]) w=weight[closer] <= weight_sort[self.optimize_frac*len(weight_sort)] # w=numpy.logical_and(w,closer) ans=-moment(y[closer[w]],moment=4).item() sd= y[closer[w]].std() print self.catastrophe_cut,self.eps_par,self.mask_var,ans,sd return ans
def get_moments(scores): return first_four_moments(scores) + [ mstats.moment(scores, moment=i) for i in xrange(5, 11) ]
def calib_steps(self, cps, amps, pha, nexp, expflag=None): """ Short Summary ------------- Calculates closure phase and mean squared visibilities & standard error, and apply the exposure flag. Parameters ---------- cps: 1D float array closure phases amps: 1D float array fringe visibility between each pair of holes pha: 1D float array phases nexp: integer number of exposures expflag: integer default=None number of flagged exposures Returns ------- meancp: float mean of closure phases errcp: float error of closure phases meanv2: float mean squared visibilities errv2: float mean squared error of visibilities meanpha: float mean of phases errpha: float error of phases """ # 10/14/16 Change flags exposures where vis > 1 anywhere # Apply the exposure flag expflag = None cpmask = np.zeros(cps.shape, dtype=bool) blmask = np.zeros(amps.shape, dtype=bool) if expflag is not None: nexp -= len(expflag) # don't count the bad exposures cpmask[expflag, :] = True blmask[expflag, :] = True else: pass meancp = np.ma.masked_array(cps, mask=cpmask).mean(axis=0) meanv2 = np.ma.masked_array(amps, mask=blmask).mean(axis=0)**2 meanpha = np.ma.masked_array(pha, mask=blmask).mean(axis=0)**2 errcp = np.sqrt(mstats.moment(np.ma.masked_array(cps, mask=cpmask), moment=2, axis=0)) / np.sqrt(nexp) errv2 = np.sqrt(mstats.moment(np.ma.masked_array(amps**2, mask=blmask), moment=2, axis=0)) / np.sqrt(nexp) errpha = np.sqrt(mstats.moment(np.ma.masked_array(pha, mask=blmask), moment=2, axis=0)) / np.sqrt(nexp) # Set cutoff accd to Kraus 2008 - 2/3 of median errcp[errcp < (2 / 3.0) * np.median(errcp)] = (2 / 3.0) * np.median(errcp) errpha[errpha < (2 / 3.0) * np.median(errpha)] = (2 / 3.0) * np.median(errpha) errv2[errv2 < (2 / 3.0) * np.median(errv2)] = (2 / 3.0) * np.median(errv2) return meancp, errcp, meanv2, errv2, meanpha, errpha
def moment(self, order): return mstats.moment(self, order)
keys.add(key) return compute_potential(counts, num_elements) if __name__ == '__main__': alphas = [] moments = [[], [], [], []] for alpha in (num/10. for num in xrange(1, 51)): potentials = [get_potential(alpha) for _ in range(100)] print alpha alphas.append(alpha) moments[0].append(np.mean(potentials)) moments[1].append(np.std(potentials)) moments[2].append(moment(potentials, 3)) moments[3].append(moment(potentials, 4)) plt.figure() plt.xlabel('Load Factor') plt.ylabel('Potential') plt.title('Potential as a function of Load Factor') plt.plot(alphas, moments[0]) plt.savefig('mean_100_50.png') plt.figure() plt.plot(moments[1]) plt.savefig('std_100_50.png') slope, intercept, r_val, p_val, std_err = linregress(alphas, moments[0]) print slope, intercept, r_val, p_val, std_err
def get_moments(scores): return first_four_moments(scores) + [mstats.moment(scores,moment=i) for i in xrange(5,11)]
def get_sleep_survey_data(individual_timeline_directory): colunms = [ 'date', 'start_recording_time', 'end_recording_time', 'type', 'shift_type', 'is_sleep_before_work', 'is_sleep_after_work', 'is_sleep_after_sleep', 'is_sleep_transition_before_work', 'is_sleep_transition_after_work', 'SleepMinutesAwake', 'SleepMinutesStageDeep', 'SleepMinutesStageLight', 'SleepMinutesStageRem', 'SleepMinutesStageWake', 'SleepEfficiency', 'work_status', 'duration_in_seconds', 'job_survey_timestamp', 'health_survey_timestamp', 'personality_survey_timestamp', 'itp_mgt', 'irb_mgt', 'ocb_mgt', 'cwb_mgt', 'neu_mgt', 'con_mgt', 'ext_mgt', 'agr_mgt', 'ope_mgt', 'pos_af_mgt', 'neg_af_mgt', 'anxiety_mgt', 'stress_mgt', 'alcohol_mgt', 'tobacco_mgt', 'exercise_mgt', 'sleep_mgt', 'interaction_mgt', 'activity_mgt', 'location_mgt', 'event_mgt', 'work_mgt' ] output_colunms = [ 'participant_id', 'sleep_transition_number', 'night_shift_type', 'nurse_years', 'general_health', 'life_satisfaction', 'nap_proxy', 'switcher_sleep', 'night_stay', 'incomplete_shifter', 'date', 'start_recording_time', 'end_recording_time', 'sleep_transition', 'type', 'shift_type', 'is_sleep_before_work', 'is_sleep_after_work', 'is_sleep_after_sleep', 'is_sleep_transition_before_work', 'is_sleep_transition_after_work', 'SleepMinutesAwake', 'SleepMinutesStageDeep', 'SleepMinutesStageLight', 'SleepMinutesStageRem', 'SleepMinutesStageWake', 'SleepEfficiency', 'work_status', 'duration_in_seconds', 'job_survey_timestamp', 'health_survey_timestamp', 'personality_survey_timestamp', 'itp_mgt', 'irb_mgt', 'ocb_mgt', 'cwb_mgt', 'neu_mgt', 'con_mgt', 'ext_mgt', 'agr_mgt', 'ope_mgt', 'pos_af_mgt', 'neg_af_mgt', 'anxiety_mgt', 'stress_mgt', 'alcohol_mgt', 'tobacco_mgt', 'exercise_mgt', 'sleep_mgt', 'interaction_mgt', 'activity_mgt', 'location_mgt', 'event_mgt', 'work_mgt', 'sleep_heart_rate_max', 'sleep_heart_rate_min', 'sleep_heart_rate_mean', 'sleep_heart_rate_std', 'sleep_heart_rate_percentile_10', 'sleep_heart_rate_percentile_90', 'sleep_heart_rate_kurtosis', 'sleep_heart_rate_moment', 'before_sleep_heart_rate_max', 'before_sleep_heart_rate_min', 'before_sleep_heart_rate_mean', 'before_sleep_heart_rate_std', 'before_sleep_heart_rate_percentile_10', 'before_sleep_heart_rate_percentile_90', 'before_sleep_heart_rate_kurtosis', 'before_sleep_heart_rate_moment', 'after_sleep_heart_rate_max', 'after_sleep_heart_rate_min', 'after_sleep_heart_rate_mean', 'after_sleep_heart_rate_std', 'after_sleep_heart_rate_percentile_10', 'after_sleep_heart_rate_percentile_90', 'after_sleep_heart_rate_kurtosis', 'after_sleep_heart_rate_moment' ] shift_type = ['day', 'night', 'unknown'] # Get participant ID # IDs = getParticipantID(main_data_directory) participant_info = getParticipantInfo(main_data_directory) participant_info = participant_info.set_index('MitreID') # Read Pre-Study info PreStudyInfo = read_pre_study_info(main_data_directory) final_df = pd.DataFrame() # Heart rate data # Assume the duration is 600 minute maximum heart_rate_df = pd.DataFrame() # Append colunms for i in range(sleep_duration_max_in_minute): output_colunms.append(str('minute_' + str(i))) for i in range(before_sleep_duration_max_in_minute): output_colunms.append(str('before_minute_' + str(i))) for i in range(after_sleep_duration_max_in_minute): output_colunms.append(str('after_minute_' + str(i))) # Read final df if os.path.exists(os.path.join('output', 'sleep_survey_full.csv')) is True: final_df = pd.read_csv(os.path.join('output', 'sleep_survey_full.csv')) else: # Read timeline for each participant for user_id in participant_info.index: participant_id = participant_info.loc[user_id]['ParticipantID'] shift = 1 if participant_info.loc[user_id][ 'Shift'] == 'Day shift' else 2 wave_number = participant_info.loc[user_id]['Wave'] user_pre_study_info = PreStudyInfo.loc[PreStudyInfo['uid'] == user_id] if wave_number != 3: print('Start Processing (Individual timeline): ' + participant_id) file_name = os.path.join( main_data_directory, 'keck_wave2/3_preprocessed_data/fitbit', participant_id + '_heartRate.csv') fitbit_data_df = pd.DataFrame() if os.path.exists(file_name) is True: fitbit_data_df = getDataFrame(file_name) fitbit_data_df.index = pd.to_datetime(fitbit_data_df.index) fitbit_data_df = fitbit_data_df.sort_index() timeline = pd.read_csv( os.path.join(individual_timeline_directory, shift_type[shift - 1], participant_id + '.csv')) timeline = timeline.loc[timeline['type'] == 'sleep'] if len(timeline) > 0: timeline_df = pd.DataFrame() timeline = timeline[colunms] timeline['participant_id'] = participant_id timeline['sleep_transition'] = np.nan timeline['sleep_heart_rate_max'] = np.nan timeline['sleep_heart_rate_min'] = np.nan timeline['sleep_heart_rate_mean'] = np.nan timeline['sleep_heart_rate_std'] = np.nan timeline['sleep_heart_rate_percentile_10'] = np.nan timeline['sleep_heart_rate_percentile_90'] = np.nan timeline['sleep_heart_rate_kurtosis'] = np.nan timeline['sleep_heart_rate_moment'] = np.nan timeline['before_sleep_heart_rate_max'] = np.nan timeline['before_sleep_heart_rate_min'] = np.nan timeline['before_sleep_heart_rate_mean'] = np.nan timeline['before_sleep_heart_rate_std'] = np.nan timeline['before_sleep_heart_rate_percentile_10'] = np.nan timeline['before_sleep_heart_rate_percentile_90'] = np.nan timeline['before_sleep_heart_rate_kurtosis'] = np.nan timeline['before_sleep_heart_rate_moment'] = np.nan timeline['after_sleep_heart_rate_max'] = np.nan timeline['after_sleep_heart_rate_min'] = np.nan timeline['after_sleep_heart_rate_mean'] = np.nan timeline['after_sleep_heart_rate_std'] = np.nan timeline['after_sleep_heart_rate_percentile_10'] = np.nan timeline['after_sleep_heart_rate_percentile_90'] = np.nan timeline['after_sleep_heart_rate_kurtosis'] = np.nan timeline['after_sleep_heart_rate_moment'] = np.nan timeline['sleep_transition'] = np.nan timeline['night_shift_type'] = np.nan timeline['sleep_transition_number'] = np.nan timeline['nap_proxy'] = np.nan timeline['switcher_sleep'] = np.nan timeline['night_stay'] = np.nan timeline['incomplete_shifter'] = np.nan timeline['nurse_years'] = user_pre_study_info[ 'nurse_years_pre-study'].values[0] if len( user_pre_study_info['nurse_years_pre-study'] ) > 0 else np.nan timeline['general_health'] = user_pre_study_info[ 'general_health_pre-study'].values[0] if len( user_pre_study_info['general_health_pre-study'] ) > 0 else np.nan timeline['life_satisfaction'] = user_pre_study_info[ 'life_satisfaction_pre-study'].values[0] if len( user_pre_study_info['life_satisfaction_pre-study'] ) > 0 else np.nan for i in range(sleep_duration_max_in_minute): timeline['minute_' + str(i)] = np.nan if len(fitbit_data_df) > 0: for index, row in timeline.iterrows(): print('Process: ' + pd.to_datetime(row['start_recording_time']). strftime(date_only_date_time_format)) # Initialize start and end recording time start_recording_time = row['start_recording_time'] end_recording_time = row['end_recording_time'] # decide which type of switch the participant is # 3: Switcher sleep # 5: Incomplete switcher # 2: Nap Proxy # 4: No Sleep if row['shift_type'] == 2 and row[ 'is_sleep_transition_before_work'] == 1: start_recording_datetime = pd.to_datetime( row['start_recording_time']) end_recording_datetime = pd.to_datetime( row['end_recording_time']) if 2 <= start_recording_datetime.hour <= 18: if (end_recording_datetime - start_recording_datetime ).total_seconds() > 3600 * 6: row['sleep_transition'] = 5 else: row['sleep_transition'] = 2 elif start_recording_datetime.hour <= 1 or start_recording_datetime.hour >= 20: if (end_recording_datetime - start_recording_datetime ).total_seconds() > 3600 * 6: row['sleep_transition'] = 3 else: row['sleep_transition'] = 4 else: row['sleep_transition'] = 4 # Fitbit stat during the sleep # heart rate: mean, min, max, std, etc. fitbit_data_df['HeartRate'] = fitbit_data_df[ 'HeartRate'].apply(pd.to_numeric) fitbit_row_df = pd.DataFrame() fitbit_row_df = fitbit_data_df[ start_recording_time:end_recording_time] end_recording_time_after_sleep = ( pd.to_datetime(end_recording_time) + timedelta( hours=2)).strftime(date_time_format)[:-3] start_recording_time_before_sleep = ( pd.to_datetime(start_recording_time) - timedelta(hours=2) ).strftime(date_time_format)[:-3] fitbit_row_before_sleep_df = fitbit_data_df[ start_recording_time_before_sleep: start_recording_time] fitbit_row_after_sleep_df = fitbit_data_df[ end_recording_time: end_recording_time_after_sleep] if len(fitbit_row_df) > 150: row['sleep_heart_rate_max'] = np.max( np.array(fitbit_row_df, dtype=int)) row['sleep_heart_rate_min'] = np.min( np.array(fitbit_row_df, dtype=int)) row['sleep_heart_rate_mean'] = np.mean( np.array(fitbit_row_df, dtype=int)) row['sleep_heart_rate_std'] = np.std( np.array(fitbit_row_df, dtype=int)) row['sleep_heart_rate_percentile_10'] = np.percentile( np.array(fitbit_row_df, dtype=int), 10) row['sleep_heart_rate_percentile_90'] = np.percentile( np.array(fitbit_row_df, dtype=int), 90) row['sleep_heart_rate_kurtosis'] = kurtosis( np.array(fitbit_row_df, dtype=int))[0] row['sleep_heart_rate_moment'] = moment( np.array(fitbit_row_df, dtype=int))[0] # We want to get minute level HR as well # heart_rate_frame_df = pd.DataFrame(columns=heart_rate_df_col) start_time = pd.to_datetime( start_recording_time).replace(second=0) for i in range(0, sleep_duration_max_in_minute, minute_offset): minute_start = pd.to_datetime( start_time) + timedelta(minutes=i) minute_end = pd.to_datetime( start_time) + timedelta(minutes=i + 1) heart_rate_minute_data = pd.DataFrame() if (pd.to_datetime(end_recording_time) - minute_end).total_seconds() > 0: heart_rate_minute_data = fitbit_data_df[ minute_start:minute_end] if len(heart_rate_minute_data) > 0: heart_rate_minute_data = heart_rate_minute_data.loc[ (heart_rate_minute_data[ 'HeartRate'] <= 180) & (heart_rate_minute_data[ 'HeartRate'] >= 30)] row['minute_' + str(i)] = np.mean( fitbit_data_df[minute_start:minute_end] .values) if len(heart_rate_minute_data ) > 0 else np.nan # Fitbit data before sleep if len(fitbit_row_before_sleep_df) > 100: row['before_sleep_heart_rate_max'] = np.max( np.array(fitbit_row_before_sleep_df)) row['before_sleep_heart_rate_min'] = np.min( np.array(fitbit_row_before_sleep_df)) row['before_sleep_heart_rate_mean'] = np.mean( np.array(fitbit_row_before_sleep_df)) row['before_sleep_heart_rate_std'] = np.std( np.array(fitbit_row_before_sleep_df)) row['before_sleep_heart_rate_percentile_10'] = np.percentile( np.array(fitbit_row_before_sleep_df), 10) row['before_sleep_heart_rate_percentile_90'] = np.percentile( np.array(fitbit_row_before_sleep_df), 90) row['before_sleep_heart_rate_kurtosis'] = kurtosis( np.array(fitbit_row_before_sleep_df))[0] row['before_sleep_heart_rate_moment'] = moment( np.array(fitbit_row_before_sleep_df), moment=1)[0] # We want to get minute level HR as well # heart_rate_frame_df = pd.DataFrame(columns=heart_rate_df_col) start_time = pd.to_datetime( start_recording_time_before_sleep).replace( second=0) for i in range( before_sleep_duration_max_in_minute): minute_start = pd.to_datetime( start_time) + timedelta(minutes=i) minute_end = pd.to_datetime( start_time) + timedelta(minutes=i + 1) heart_rate_minute_data = pd.DataFrame() if (pd.to_datetime(start_recording_time) - minute_end).total_seconds() > 0: heart_rate_minute_data = fitbit_row_before_sleep_df[ minute_start:minute_end] if len(heart_rate_minute_data) > 0: heart_rate_minute_data = heart_rate_minute_data.loc[ (heart_rate_minute_data[ 'HeartRate'] <= 180) & (heart_rate_minute_data[ 'HeartRate'] >= 30)] row['before_minute_' + str(i)] = np.mean( heart_rate_minute_data[ minute_start:minute_end].values ) if len( heart_rate_minute_data) > 0 else np.nan if len(fitbit_row_after_sleep_df) > 100: row['after_sleep_heart_rate_max'] = np.max( np.array(fitbit_row_after_sleep_df)) row['after_sleep_heart_rate_min'] = np.min( np.array(fitbit_row_after_sleep_df)) row['after_sleep_heart_rate_mean'] = np.mean( np.array(fitbit_row_after_sleep_df)) row['after_sleep_heart_rate_std'] = np.std( np.array(fitbit_row_after_sleep_df)) row['after_sleep_heart_rate_percentile_10'] = np.percentile( np.array(fitbit_row_after_sleep_df), 10) row['after_sleep_heart_rate_percentile_90'] = np.percentile( np.array(fitbit_row_after_sleep_df), 90) row['after_sleep_heart_rate_kurtosis'] = kurtosis( np.array(fitbit_row_after_sleep_df))[0] row['after_sleep_heart_rate_moment'] = moment( np.array(fitbit_row_after_sleep_df), moment=1)[0] # We want to get minute level HR as well # heart_rate_frame_df = pd.DataFrame(columns=heart_rate_df_col) start_time = pd.to_datetime( end_recording_time).replace(second=0) for i in range( after_sleep_duration_max_in_minute): minute_start = pd.to_datetime( start_time) + timedelta(minutes=i) minute_end = pd.to_datetime( start_time) + timedelta(minutes=i + 1) heart_rate_minute_data = pd.DataFrame() if (pd.to_datetime( end_recording_time_after_sleep) - minute_end).total_seconds() > 0: heart_rate_minute_data = fitbit_row_after_sleep_df[ minute_start:minute_end] if len(heart_rate_minute_data) > 0: heart_rate_minute_data = heart_rate_minute_data.loc[ (heart_rate_minute_data[ 'HeartRate'] <= 180) & (heart_rate_minute_data[ 'HeartRate'] >= 30)] row['after_minute_' + str(i)] = np.mean( heart_rate_minute_data[ minute_start:minute_end].values ) if len( heart_rate_minute_data) > 0 else np.nan timeline_df = timeline_df.append(row) else: timeline_df = timeline if len(timeline_df) > 0: number_of_transition = len(timeline_df.loc[ timeline_df['sleep_transition'] > 0]) if number_of_transition > 0: timeline_df[ 'sleep_transition_number'] = number_of_transition night_shift_distribution = [ len(timeline_df.loc[ timeline_df['sleep_transition'] == 2]), len(timeline_df.loc[ timeline_df['sleep_transition'] == 3]), len(timeline_df.loc[ timeline_df['sleep_transition'] == 4]), len(timeline_df.loc[ timeline_df['sleep_transition'] == 5]) ] timeline_df[ 'nap_proxy'] = night_shift_distribution[0] timeline_df[ 'switcher_sleep'] = night_shift_distribution[1] timeline_df[ 'night_stay'] = night_shift_distribution[2] timeline_df[ 'incomplete_shifter'] = night_shift_distribution[ 3] timeline_df['night_shift_type'] = np.argmax( night_shift_distribution) + 2 timeline_df = timeline_df[output_colunms] final_df = final_df.append(timeline_df) final_df = final_df[output_colunms] final_df.to_csv(os.path.join('../output', 'sleep_survey_full.csv'), index=False) # final_df = final_df[output_colunms] # final_df.to_csv(os.path.join('output', 'sleep_survey_full.csv'), index=False) return final_df
def Moment(data, moment_n): return scimom.moment(data, moment_n)
def Moment(data, moment_n): return scimom.moment(data, moment_n)