def test_resample_by_depth(self): inc_amt =0.001 stat = ['mean'] df, = resample('d',small_file,stat,inc_amt,'depth (m abs)',False) expected_result = clean_data(load_csv(output_small_file_depth)) expected_result = expected_result.set_index(depth_header.label) assert_frame_equal(df,expected_result) df1 = resample('d',small_file,stat,inc_amt,None,False) assert_frame_equal(df1[1],expected_result) self.assertEqual(df1[0].index.name, depth_header2.label) df2 = resample('d',small_file,None,inc_amt,None,False) expected_result2 = clean_data(load_csv(output_small_file_stat)) expected_result2 = expected_result2.set_index(depth_header.label) assert_frame_equal(df2[1],expected_result2) self.assertEqual(df2[0].index.name, depth_header2.label) df = resample('D',small_file,None,inc_amt,None,False) assert_frame_equal(df[1],expected_result2) df = resample('Depth',small_file,None,inc_amt,None,False) assert_frame_equal(df[1],expected_result2) df = resample('depth',small_file,None,inc_amt,None,False) assert_frame_equal(df[1],expected_result2)
def test_create_range_for_depths_inc(self): inc_amt = 0.005 expected_result = [0.605, 0.610, 0.615, 0.620] input_test = load_csv(os.path.join('csv_files', 'input_depths.csv')) input_test = clean_data(input_test) result = create_range_for_depths(input_test.loc[:, 'depth (m we)'].values.tolist(), inc_amt) self.assertEqual(expected_result, result)
def test_create_range_for_depths_inc_decimals(self): inc_amt = 0.0005 expected_result = [0.6051, 0.6056, 0.6061, 0.6066, 0.6071, 0.6076, 0.6081, 0.6086, 0.6091, 0.6096] input_test = load_csv(os.path.join('csv_files', 'input_depth_decimal.csv')) input_test = clean_data(input_test) result = create_range_for_depths(input_test.loc[:, 'depth (m we) '].values.tolist(), inc_amt) self.assertEqual(expected_result, result)
def test_by_year_stats(self): inc_amt = 1 df = by_years(dc,year_header,inc_amt) expected_result = clean_data(load_csv(output_small_year_stat)) expected_result = expected_result.set_index(year_header.label) assert_frame_equal(df, expected_result)
def test_create_range_for_depths_2(self): inc_amt = 0.01 expected_result = [1.64, 1.65, 1.66, 1.67, 1.68, 1.69] input_test = load_csv(os.path.join('csv_files', 'input_depths.csv')) input_test = clean_data(input_test) result = create_range_for_depths(input_test.loc[:, 'depth (m abs)'].values.tolist(), inc_amt) self.assertEqual(expected_result, result)
def test_by_depths_stats(self): inc_amt = 0.001 stat = None df = by_depths(dc,depth_header,inc_amt,stat) expected_result = clean_data(load_csv(output_small_file_stat)) expected_result = expected_result.set_index(depth_header.label) assert_frame_equal(df, expected_result)
def test_by_years(self): df = by_years(dc,year_header,1,['mean'],year_header) expected_result = clean_data(load_csv(output_small_file)) expected_result = expected_result.set_index('Year_Dat210617_(CE)') assert_frame_equal(df, expected_result) df2 = by_years(dc,year_header,2,['mean'],year_header) expected_result2 = clean_data(load_csv(output_small_file2)) expected_result2 = expected_result2.set_index('Year_Dat210617_(CE)') assert_frame_equal(df2, expected_result2) dfm = by_years(dc,year_header,1,['max','std'],year_header) expected_resultm = clean_data(load_csv(output_small_filem)) expected_resultm = expected_resultm.set_index('Year_Dat210617_(CE)') assert_frame_equal(dfm, expected_resultm)
def test_clean_LAICPMS_data(self): df = process_laser_data(f, depth_age_file) # to_csv(f.dirname,df,'process_laser_data_test.csv',False) expected_result = load_csv(test_process_laser) self.assertEqual(df.columns[0], 'depth (m abs)') self.assertEqual(df.columns[1], 'year') assert_frame_equal(expected_result, df) self.assertEqual(expected_result.shape[0], df.shape[0])
def test_resample_by(self): depth_header, = process_header_str('depth (m abs)') df, =resample_by(f_HR,f_LR,['mean'],'depth (m abs)',False) expected_result = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1_mean.csv')).astype(float) expected_result = expected_result.set_index(depth_header.label) assert_frame_equal(df,expected_result) df2, =resample_by(f_HR,f_LR,['mean','std'],'depth (m abs)',False) expected_result2 = load_csv(os.path.join('csv_files','output','test_input_dd_2_resample_by_test_input_dd_1_depth_abs_(m)_mean_std.csv')).astype(float) expected_result2 = expected_result2.set_index(depth_header.label) assert_frame_equal(df2,expected_result2) df1, =resample_by(f_HR,f_LR,None,'depth (m abs)',False) expected_result1 = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1.csv')).astype(float) expected_result1 = expected_result1.set_index(depth_header.label) assert_frame_equal(df1,expected_result1) df1 =resample_by(f_HR,f_LR,None,None,False) expected_result1 = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1.csv')).astype(float) expected_result1 = expected_result1.set_index(depth_header.label) assert_frame_equal(df1[0],expected_result1)
def test_add_year_column(self): df = clean_data(f.raw_data) df = df[(df['Time'] > f.washin_time) & (df['Time'] < f.laser_time - f.washout_time)] df = df.reset_index(drop=True) df = df.drop('Time', 1) df = add_depth_column(df, f.start_depth_m, f.end_depth_m) df = add_year_column(df, depth_age_file) self.assertEqual(df.columns[1], 'year') df = df.round(5) expected_result = load_csv(test_process_laser) self.assertEqual(df.columns[0], 'depth (m abs)') assert_frame_equal(expected_result, df)
def test_resample(self): inc_amt = 1 stat = ['mean'] df = resample('y',small_file,stat,inc_amt,None,False) expected_result = clean_data(load_csv(output_small_file)) expected_result = expected_result.set_index('Year_Dat210617_(CE)') assert_frame_equal(df[0],expected_result) self.assertEqual(df[1].index.name, 'Year_Dat011216V2_(CE)') df, = resample('y',small_file,stat,inc_amt,'Dat210617',False) assert_frame_equal(df,expected_result) df, = resample('Y',small_file,stat,inc_amt,'Dat210617',False) assert_frame_equal(df,expected_result) df, = resample('Year',small_file,stat,inc_amt,'Dat210617',False) assert_frame_equal(df,expected_result) df, = resample('year',small_file,stat,inc_amt,'Dat210617',False) assert_frame_equal(df,expected_result)
def testReadCSVFile(self): frame = load_csv(os.path.join('csv_files','input', 'small.csv')) self.assertEqual('Dat210617', frame.columns[0], 'First column should be Dat210617') self.assertEqual(2009.8, frame.values[9][0], "Last value should be 2011.6986319241")
def test_readme_output_laser_file(self): input_laser_template=\ """ ReadMeFile CCI-Data-Processor Authors: Mark Royer and Heather Clifford Date ran: {run_date} Process: Process and Compile {type} LA-ICP-MS Data Directory Folder: {directory} Prefix of Cores: {prefix} Cores in Directory = {folders} Depth Age File = {depth_age_file} *** Additional Information about Individual Cores & Runs found in file: {info_file} Directory Information: Year: {years} Year Range: {year_max} - {year_min} Depth: {depths} Depth Range: {depth_min} - {depth_max} Resolution: {resolution} Samples: {samples} Output Files: {csv_filename} """ directory=os.path.join('csv_files','test_directory') dc = FrameClass(load_csv(os.path.join(directory,'csv_files','LA-ICP-MS_raw_LR.csv'))) dc.df =dc.df.set_index(['depth (m abs)']) resolution = 'Medium' prefix='KCC' depth_age_file = 'depth_age_file.csv' info_file ='info_file.csv' run_date='2017-08-03' file ='LA-ICP-MS_raw_LR.csv' output_type = 'Raw' result=readme_laser_file(input_laser_template,directory,prefix,depth_age_file, dc, resolution, run_date,info_file, file,output_type) expected_result=\ """ ReadMeFile CCI-Data-Processor Authors: Mark Royer and Heather Clifford Date ran: 2017-08-03 Process: Process and Compile Raw LA-ICP-MS Data Directory Folder: test_directory Prefix of Cores: KCC Cores in Directory = KCC1, KCC2 Depth Age File = depth_age_file.csv *** Additional Information about Individual Cores & Runs found in file: info_file.csv Directory Information: Year: year Year Range: 676 - 654 Depth: depth (m abs) Depth Range: 60.085 - 60.255 Resolution: Medium Samples: Na23, Mg25, Cu63, Pb208, Sn118 Output Files: LA-ICP-MS_raw_LR.csv """ self.assertEqual(result,expected_result)
def test_normalize_data(self): input_df = load_csv(os.path.join('csv_files', 'normalize_input.csv')) input_df=clean_data(input_df) result = normalize_data(input_df) expected_result = load_csv(os.path.join('csv_files', 'normalize_output.csv')) assert_frame_equal(expected_result,result)
from climatechange.common_functions import DataClass, clean_data,\ load_csv, to_csv # dc = DataClass(os.path.join('csv_files','input', 'small.csv')) small_file = os.path.join('csv_files','input', 'small.csv') dc = DataClass(small_file) output_small_file = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_mean.csv') output_small_file2 = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_2_year_resolution_for_mean.csv') output_small_filem = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_max_std.csv') output_small_file_depth = os.path.join('csv_files','output','small_resample_by_0.001_depth_abs_(m)_mean.csv') output_small_file_01 = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_max_count.csv') output_small_file_stat = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_stat.csv') output_small_year_stat = os.path.join('csv_files','output','small_resample_by_1_year_stat.csv') input_test_zeros_and_numbers = clean_data(load_csv(os.path.join('csv_files', 'input_test_zeros_and_numbers.csv'))) output_by_LR = os.path.join('csv_files','output','resample_by_LR_output.csv') f_HR = os.path.join('csv_files','input', 'test_input_dd_2.csv') f_LR = os.path.join('csv_files','input', 'test_input_dd_1.csv') f_empty = os.path.join('csv_files','input', 'test_input_dd_empty.csv') dc_LR = DataClass(f_LR) dc_HR = DataClass(f_HR) depth_header, = process_header_str('depth (m abs)') depth_header2, = process_header_str('depth (m we)') year_header,=process_header_str('Dat210617') year_header2,=process_header_str('Dat011216V2') class Test(unittest.TestCase):