def test_resample_by_depth(self):
inc_amt =0.001
stat = ['mean']
df, = resample('d',small_file,stat,inc_amt,'depth (m abs)',False)
expected_result = clean_data(load_csv(output_small_file_depth))
expected_result = expected_result.set_index(depth_header.label)
assert_frame_equal(df,expected_result)
df1 = resample('d',small_file,stat,inc_amt,None,False)
assert_frame_equal(df1[1],expected_result)
self.assertEqual(df1[0].index.name, depth_header2.label)
df2 = resample('d',small_file,None,inc_amt,None,False)
expected_result2 = clean_data(load_csv(output_small_file_stat))
expected_result2 = expected_result2.set_index(depth_header.label)
assert_frame_equal(df2[1],expected_result2)
self.assertEqual(df2[0].index.name, depth_header2.label)
df = resample('D',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
df = resample('Depth',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
df = resample('depth',small_file,None,inc_amt,None,False)
assert_frame_equal(df[1],expected_result2)
def test_create_range_for_depths_inc_decimals(self):
inc_amt = 0.0005
expected_result = [0.6051, 0.6056, 0.6061, 0.6066, 0.6071, 0.6076, 0.6081, 0.6086, 0.6091, 0.6096]
input_test = load_csv(os.path.join('csv_files', 'input_depth_decimal.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m we) '].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_create_range_for_depths_2(self):
inc_amt = 0.01
expected_result = [1.64, 1.65, 1.66, 1.67, 1.68, 1.69]
input_test = load_csv(os.path.join('csv_files', 'input_depths.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m abs)'].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_create_range_for_depths_inc(self):
inc_amt = 0.005
expected_result = [0.605, 0.610, 0.615, 0.620]
input_test = load_csv(os.path.join('csv_files', 'input_depths.csv'))
input_test = clean_data(input_test)
result = create_range_for_depths(input_test.loc[:, 'depth (m we)'].values.tolist(), inc_amt)
self.assertEqual(expected_result, result)
def test_by_year_stats(self):
inc_amt = 1
df = by_years(dc,year_header,inc_amt)
expected_result = clean_data(load_csv(output_small_year_stat))
expected_result = expected_result.set_index(year_header.label)
assert_frame_equal(df, expected_result)
def test_by_depths_stats(self):
inc_amt = 0.001
stat = None
df = by_depths(dc,depth_header,inc_amt,stat)
expected_result = clean_data(load_csv(output_small_file_stat))
expected_result = expected_result.set_index(depth_header.label)
assert_frame_equal(df, expected_result)
def test_by_years(self):
df = by_years(dc,year_header,1,['mean'],year_header)
expected_result = clean_data(load_csv(output_small_file))
expected_result = expected_result.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df, expected_result)
df2 = by_years(dc,year_header,2,['mean'],year_header)
expected_result2 = clean_data(load_csv(output_small_file2))
expected_result2 = expected_result2.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df2, expected_result2)
dfm = by_years(dc,year_header,1,['max','std'],year_header)
expected_resultm = clean_data(load_csv(output_small_filem))
expected_resultm = expected_resultm.set_index('Year_Dat210617_(CE)')
assert_frame_equal(dfm, expected_resultm)
def test_clean_data(self):
input_test = DataFrame([['str', 0, 1],
[4, 5, 6],
[7, 'str', 0]])
expected_output = DataFrame([[nan, nan, 1],
[4, 5, 6],
[7, nan, nan]])
result = clean_data(input_test)
assert_frame_equal(expected_output, result)
def process_laser_data(f: LaserInput, depth_age_file) -> DataFrame:
df = clean_data(f.raw_data)
df = df[(df['Time'] > f.washin_time)
& (df['Time'] < f.laser_time - f.washout_time)]
df = df.reset_index(drop=True)
df = df.drop('Time', 1)
df = add_depth_column(df, f.start_depth_m, f.end_depth_m)
df = add_year_column(df, depth_age_file)
df = df.round(5)
return df
def test_add_year_column(self):
df = clean_data(f.raw_data)
df = df[(df['Time'] > f.washin_time)
& (df['Time'] < f.laser_time - f.washout_time)]
df = df.reset_index(drop=True)
df = df.drop('Time', 1)
df = add_depth_column(df, f.start_depth_m, f.end_depth_m)
df = add_year_column(df, depth_age_file)
self.assertEqual(df.columns[1], 'year')
df = df.round(5)
expected_result = load_csv(test_process_laser)
self.assertEqual(df.columns[0], 'depth (m abs)')
assert_frame_equal(expected_result, df)
def test_resample(self):
inc_amt = 1
stat = ['mean']
df = resample('y',small_file,stat,inc_amt,None,False)
expected_result = clean_data(load_csv(output_small_file))
expected_result = expected_result.set_index('Year_Dat210617_(CE)')
assert_frame_equal(df[0],expected_result)
self.assertEqual(df[1].index.name, 'Year_Dat011216V2_(CE)')
df, = resample('y',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('Y',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('Year',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
df, = resample('year',small_file,stat,inc_amt,'Dat210617',False)
assert_frame_equal(df,expected_result)
def test_normalize_data(self):
input_df = load_csv(os.path.join('csv_files', 'normalize_input.csv'))
input_df=clean_data(input_df)
result = normalize_data(input_df)
expected_result = load_csv(os.path.join('csv_files', 'normalize_output.csv'))
assert_frame_equal(expected_result,result)
from climatechange.common_functions import DataClass, clean_data,\
load_csv, to_csv
# dc = DataClass(os.path.join('csv_files','input', 'small.csv'))
small_file = os.path.join('csv_files','input', 'small.csv')
dc = DataClass(small_file)
output_small_file = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_mean.csv')
output_small_file2 = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_2_year_resolution_for_mean.csv')
output_small_filem = os.path.join('csv_files','output','Year_Dat210617_(CE)_resampled_by_1_year_resolution_for_max_std.csv')
output_small_file_depth = os.path.join('csv_files','output','small_resample_by_0.001_depth_abs_(m)_mean.csv')
output_small_file_01 = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_max_count.csv')
output_small_file_stat = os.path.join('csv_files','output','small_resample_by_0.01_depth_abs_(m)_stat.csv')
output_small_year_stat = os.path.join('csv_files','output','small_resample_by_1_year_stat.csv')
input_test_zeros_and_numbers = clean_data(load_csv(os.path.join('csv_files', 'input_test_zeros_and_numbers.csv')))
output_by_LR = os.path.join('csv_files','output','resample_by_LR_output.csv')
f_HR = os.path.join('csv_files','input', 'test_input_dd_2.csv')
f_LR = os.path.join('csv_files','input', 'test_input_dd_1.csv')
f_empty = os.path.join('csv_files','input', 'test_input_dd_empty.csv')
dc_LR = DataClass(f_LR)
dc_HR = DataClass(f_HR)
depth_header, = process_header_str('depth (m abs)')
depth_header2, = process_header_str('depth (m we)')
year_header,=process_header_str('Dat210617')
year_header2,=process_header_str('Dat011216V2')
class Test(unittest.TestCase):
