def test_resample_by_depth(self):
inc_amt =0.001
stat = ['mean']
df, = resample('d',small_file,stat,inc_amt,'depth (m abs)',False)
expected_result = clean_data(load_csv(output_small_file_depth))
expected_result = expected_result.set_index(depth_header.label)
assert_frame_equal(df,expected_result)
df1 = resample('d',small_file,stat,inc_amt,None,False)
assert_frame_equal(df1[1],expected_result)
self.assertEqual(df1[0].index.name, depth_header2.label)
df2 = resample('d',small_file,None,inc_amt,None,False)
expected_result2 = clean_data(load_csv(output_small_file_stat))
expected_result2 = expected_result2.set_index(depth_header.label)
assert_frame_equal(df2[1],expected_result2)
self.assertEqual(df2[0].index.name, depth_header2.label)
df = resample('D',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
df = resample('Depth',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
df = resample('depth',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
def test_create_range_for_depths_inc(self):
inc_amt = 0.005
expected_result = [0.605, 0.610, 0.615, 0.620]
input_test = load_csv(os.path.join('csv_files', 'input_depths.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m we)'].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_create_range_for_depths_inc_decimals(self):
inc_amt = 0.0005
expected_result = [0.6051, 0.6056, 0.6061, 0.6066, 0.6071, 0.6076, 0.6081, 0.6086, 0.6091, 0.6096]
input_test = load_csv(os.path.join('csv_files', 'input_depth_decimal.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m we) '].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_by_year_stats(self):
inc_amt = 1
df = by_years(dc,year_header,inc_amt)
expected_result = clean_data(load_csv(output_small_year_stat))
expected_result = expected_result.set_index(year_header.label)
assert_frame_equal(df, expected_result)
def test_create_range_for_depths_2(self):
inc_amt = 0.01
expected_result = [1.64, 1.65, 1.66, 1.67, 1.68, 1.69]
input_test = load_csv(os.path.join('csv_files', 'input_depths.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m abs)'].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_by_depths_stats(self):
inc_amt = 0.001
stat = None
df = by_depths(dc,depth_header,inc_amt,stat)
expected_result = clean_data(load_csv(output_small_file_stat))
expected_result = expected_result.set_index(depth_header.label)
assert_frame_equal(df, expected_result)
def test_by_years(self):
df = by_years(dc,year_header,1,['mean'],year_header)
expected_result = clean_data(load_csv(output_small_file))
expected_result = expected_result.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df, expected_result)
df2 = by_years(dc,year_header,2,['mean'],year_header)
expected_result2 = clean_data(load_csv(output_small_file2))
expected_result2 = expected_result2.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df2, expected_result2)
dfm = by_years(dc,year_header,1,['max','std'],year_header)
expected_resultm = clean_data(load_csv(output_small_filem))
expected_resultm = expected_resultm.set_index('Year_Dat210617_(CE)')
assert_frame_equal(dfm, expected_resultm)
def test_clean_LAICPMS_data(self):
df = process_laser_data(f, depth_age_file)
# to_csv(f.dirname,df,'process_laser_data_test.csv',False)
expected_result = load_csv(test_process_laser)
self.assertEqual(df.columns[0], 'depth (m abs)')
self.assertEqual(df.columns[1], 'year')
assert_frame_equal(expected_result, df)
self.assertEqual(expected_result.shape[0], df.shape[0])
def test_resample_by(self):
depth_header, = process_header_str('depth (m abs)')
df, =resample_by(f_HR,f_LR,['mean'],'depth (m abs)',False)
expected_result = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1_mean.csv')).astype(float)
expected_result = expected_result.set_index(depth_header.label)
assert_frame_equal(df,expected_result)
df2, =resample_by(f_HR,f_LR,['mean','std'],'depth (m abs)',False)
expected_result2 = load_csv(os.path.join('csv_files','output','test_input_dd_2_resample_by_test_input_dd_1_depth_abs_(m)_mean_std.csv')).astype(float)
expected_result2 = expected_result2.set_index(depth_header.label)
assert_frame_equal(df2,expected_result2)
df1, =resample_by(f_HR,f_LR,None,'depth (m abs)',False)
expected_result1 = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1.csv')).astype(float)
expected_result1 = expected_result1.set_index(depth_header.label)
assert_frame_equal(df1,expected_result1)
df1 =resample_by(f_HR,f_LR,None,None,False)
expected_result1 = load_csv(os.path.join('csv_files','output','test_input_dd_2_depth_abs_(m)_resample_by_test_input_dd_1.csv')).astype(float)
expected_result1 = expected_result1.set_index(depth_header.label)
assert_frame_equal(df1[0],expected_result1)
def test_add_year_column(self):
df = clean_data(f.raw_data)
df = df[(df['Time'] > f.washin_time)
& (df['Time'] < f.laser_time - f.washout_time)]
df = df.reset_index(drop=True)
df = df.drop('Time', 1)
df = add_depth_column(df, f.start_depth_m, f.end_depth_m)
df = add_year_column(df, depth_age_file)
self.assertEqual(df.columns[1], 'year')
df = df.round(5)
expected_result = load_csv(test_process_laser)
self.assertEqual(df.columns[0], 'depth (m abs)')
assert_frame_equal(expected_result, df)
def test_resample(self):
inc_amt = 1
stat = ['mean']
df = resample('y',small_file,stat,inc_amt,None,False)
expected_result = clean_data(load_csv(output_small_file))
expected_result = expected_result.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df[0],expected_result)
self.assertEqual(df[1].index.name, 'Year_Dat011216V2_(CE)')
df, = resample('y',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('Y',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('Year',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('year',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
def testReadCSVFile(self):
frame = load_csv(os.path.join('csv_files','input', 'small.csv'))
self.assertEqual('Dat210617', frame.columns[0], 'First column should be Dat210617')
self.assertEqual(2009.8, frame.values[9][0], "Last value should be 2011.6986319241")
def test_readme_output_laser_file(self):
input_laser_template=\
"""
ReadMeFile
CCI-Data-Processor
Authors: Mark Royer and Heather Clifford
Date ran: {run_date}
Process: Process and Compile {type} LA-ICP-MS Data
Directory Folder: {directory}
Prefix of Cores: {prefix}
Cores in Directory = {folders}
Depth Age File = {depth_age_file}
*** Additional Information about Individual Cores & Runs found in file:
{info_file}
Directory Information:
Year: {years}
Year Range: {year_max} - {year_min}
Depth: {depths}
Depth Range: {depth_min} - {depth_max}
Resolution: {resolution}
Samples: {samples}
Output Files:
{csv_filename}
"""
directory=os.path.join('csv_files','test_directory')
dc = FrameClass(load_csv(os.path.join(directory,'csv_files','LA-ICP-MS_raw_LR.csv')))
dc.df =dc.df.set_index(['depth (m abs)'])
resolution = 'Medium'
prefix='KCC'
depth_age_file = 'depth_age_file.csv'
info_file ='info_file.csv'
run_date='2017-08-03'
file ='LA-ICP-MS_raw_LR.csv'
output_type = 'Raw'
result=readme_laser_file(input_laser_template,directory,prefix,depth_age_file, dc, resolution, run_date,info_file, file,output_type)
expected_result=\
"""
ReadMeFile
CCI-Data-Processor
Authors: Mark Royer and Heather Clifford
Date ran: 2017-08-03
Process: Process and Compile Raw LA-ICP-MS Data
Directory Folder: test_directory
Prefix of Cores: KCC
Cores in Directory = KCC1, KCC2
Depth Age File = depth_age_file.csv
*** Additional Information about Individual Cores & Runs found in file:
info_file.csv
Directory Information:
Year: year
Year Range: 676 - 654
Depth: depth (m abs)
Depth Range: 60.085 - 60.255
Resolution: Medium
Samples: Na23, Mg25, Cu63, Pb208, Sn118
Output Files:
LA-ICP-MS_raw_LR.csv
"""
self.assertEqual(result,expected_result)
def test_normalize_data(self):
input_df = load_csv(os.path.join('csv_files', 'normalize_input.csv'))
input_df=clean_data(input_df)
result = normalize_data(input_df)
expected_result = load_csv(os.path.join('csv_files', 'normalize_output.csv'))
assert_frame_equal(expected_result,result)
from climatechange.common_functions import DataClass, clean_data,\
load_csv, to_csv
# dc = DataClass(os.path.join('csv_files','input', 'small.csv'))
small_file = os.path.join('csv_files','input', 'small.csv')
dc = DataClass(small_file)
output_small_file = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_mean.csv')
output_small_file2 = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_2_year_resolution_for_mean.csv')
output_small_filem = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_max_std.csv')
output_small_file_depth = os.path.join('csv_files','output','small_resample_by_0.001_depth_abs_(m)_mean.csv')
output_small_file_01 = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_max_count.csv')
output_small_file_stat = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_stat.csv')
output_small_year_stat = os.path.join('csv_files','output','small_resample_by_1_year_stat.csv')
input_test_zeros_and_numbers = clean_data(load_csv(os.path.join('csv_files', 'input_test_zeros_and_numbers.csv')))
output_by_LR = os.path.join('csv_files','output','resample_by_LR_output.csv')
f_HR = os.path.join('csv_files','input', 'test_input_dd_2.csv')
f_LR = os.path.join('csv_files','input', 'test_input_dd_1.csv')
f_empty = os.path.join('csv_files','input', 'test_input_dd_empty.csv')
dc_LR = DataClass(f_LR)
dc_HR = DataClass(f_HR)
depth_header, = process_header_str('depth (m abs)')
depth_header2, = process_header_str('depth (m we)')
year_header,=process_header_str('Dat210617')
year_header2,=process_header_str('Dat011216V2')
class Test(unittest.TestCase):