def test06_prepareForVesperKrig(self):
csv_desc = CsvDescribe(fileTrimmed)
df_csv = csv_desc.open_pandas_dataframe()
global file_control
sub_file = os.path.splitext(os.path.basename(file_csv))[0]
file_control = sub_file + '_control_' + data_col + '.txt'
vc = VesperControl()
vc.update(xside=30, yside=30)
if not os.path.exists(file_control):
bat_file, file_control = prepare_for_vesper_krige(df_csv, data_col, file_block_txt,
TmpDir,
control_textfile=file_control,
coord_columns=[], epsg=epsg,
control_options=vc)
self.assertTrue(os.path.exists(bat_file))
self.assertTrue(os.path.exists(file_control))
self.assertTrue(os.path.exists(os.path.join(TmpDir, r'Vesper',
sub_file + '_vesperdata_' + data_col + '.csv')))
df_csv = pd.read_csv(os.path.join(TmpDir, r'Vesper',
sub_file + '_vesperdata_' + data_col + '.csv'))
x_column, y_column = predictCoordinateColumnNames(df_csv.columns)
self.assertEqual(x_column.upper(), 'EASTING')
self.assertEqual(y_column.upper(), 'NORTHING')
print('Running Vesper, Please wait....')
run_vesper(file_control)
def test_from_ISO_8859_1_csv(self):
file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")
descCSV = CsvDescribe(file)
self.assertEqual(predictCoordinateColumnNames(descCSV.get_column_names()), ['Longitude', 'Latitude'])
df = descCSV.open_pandas_dataframe()
self.assertListEqual(df.columns.to_list(), descCSV.get_column_names())
def test01_csvDescribe_ASCII(self):
csv_desc = CsvDescribe(file_csv)
self.assertEqual(csv_desc.file_encoding, 'ascii')
self.assertEqual(csv_desc.row_count, 13756)
self.assertEqual(csv_desc.column_count, 24)
self.assertEqual(predictCoordinateColumnNames(csv_desc.get_column_names()), ['Lon', 'Lat'])
self.assertTrue(csv_desc.has_column_header)
def test_csvfile_UTF8(self):
csvDesc = CsvDescribe(
os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv"))
self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1')
self.assertEqual(csvDesc.row_count, 1543)
self.assertEqual(csvDesc.column_count, 18)
self.assertEqual(
predictCoordinateColumnNames(csvDesc.get_column_names()),
['Longitude', 'Latitude'])
self.assertTrue(csvDesc.has_column_header)
def test_csvfile_ascii(self):
csvDesc = CsvDescribe(os.path.realpath(this_dir + "/data/area1_yield_ascii_wgs84.csv"))
self.assertEqual(csvDesc.file_encoding, 'ascii')
self.assertEqual(csvDesc.row_count, 13756)
self.assertEqual(csvDesc.column_count, 24)
self.assertTrue(csvDesc.has_column_header)
def test1_CreateControlHighDensity_VesperControlClass(self):
# check using VesperControl class
file_csv = os.path.realpath(this_dir + "/area2_high_trimmed.csv")
grid_file = os.path.realpath(this_dir +
"/rasters/area2_5m_blockgrid_v.txt")
data_col = r'Yld Mass(Dry)(tonne/ha)'
csv_desc = CsvDescribe(file_csv)
df_csv = csv_desc.open_pandas_dataframe()
vc = VesperControl()
vc.update(xside=30, yside=30)
global g_ctrl_file
file_bat, g_ctrl_file = prepare_for_vesper_krige(
df_csv,
data_col,
grid_file,
TEMPDIR,
control_textfile='test_high_5m_control.txt',
coord_columns=[],
epsg=28354,
control_options=vc)
if os.path.exists(kriging_ops.vesper_exe):
self.assertTrue(
os.path.exists(os.path.join(TEMPDIR, 'Vesper/Do_Vesper.bat')))
else:
self.assertEqual('', file_bat)
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_high_5m_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper',
'test_high_5m_vesperdata.csv')))
src_df = pd.read_csv(
os.path.realpath(this_dir + '/VESPER/high_5m_vesperdata.csv'))
test_df = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_high_5m_vesperdata.csv'))
pd.testing.assert_frame_equal(src_df, test_df)
with open(os.path.realpath(this_dir + '/VESPER/high_5m_control.txt')) as src_file,\
open(g_ctrl_file) as test_file:
self.assertEqual(src_file.readlines()[11:],
test_file.readlines()[11:])
def test1a_CreateControlHighDensity_block_size(self):
file_csv = os.path.realpath(this_dir + "/area2_high_trimmed.csv")
grid_file = os.path.realpath(this_dir +
"/rasters/area2_5m_blockgrid_v.txt")
data_col = r'Yld Mass(Dry)(tonne/ha)'
csv_desc = CsvDescribe(file_csv)
df_csv = csv_desc.open_pandas_dataframe()
# check using block_size argument - backwards compatible
file_bat, file_ctrl = prepare_for_vesper_krige(
df_csv,
data_col,
grid_file,
TEMPDIR,
control_textfile='test_high_5m_control.txt',
block_size=30,
coord_columns=[],
epsg=28354)
if os.path.exists(kriging_ops.vesper_exe):
self.assertTrue(
os.path.exists(os.path.join(TEMPDIR, 'Vesper/Do_Vesper.bat')))
else:
self.assertEqual('', file_bat)
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_high_5m_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper',
'test_high_5m_vesperdata.csv')))
src_df = pd.read_csv(
os.path.realpath(this_dir + '/VESPER/high_5m_vesperdata.csv'))
test_df = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_high_5m_vesperdata.csv'))
pd.testing.assert_frame_equal(src_df, test_df)
def test_csvfile_UTF8(self):
csvDesc = CsvDescribe(os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv"))
# chardet seems to detect this file as ISO-8869-9, which is described
# as "Largely the same as ISO/IEC 8859-1, replacing the rarely used
# Icelandic letters with Turkish ones."
#self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1')
self.assertEqual(csvDesc.file_encoding, 'ISO-8859-9')
self.assertEqual(csvDesc.row_count, 1543)
self.assertEqual(csvDesc.column_count, 18)
self.assertEqual(predictCoordinateColumnNames(csvDesc.get_column_names()), ['Longitude', 'Latitude'])
self.assertTrue(csvDesc.has_column_header)
self.assertEqual(csvDesc.get_column_names()[-1], csvDesc.get_alias_column_names()[-1])
self.assertNotEqual(csvDesc.get_column_names()[-2], csvDesc.get_alias_column_names()[-2])
self.assertEqual(u'Crop Flw(V)(m\xb3/s)', csvDesc.get_column_names()[-2])
self.assertEqual('CropFlw(V)(m3/s)', csvDesc.get_alias_column_names()[-2])
#check to see if unicode characters exist True if all ascii, false if not
self.assertTrue(all(ord(char) < 128 for char in csvDesc.get_alias_column_names()[-2]))
self.assertFalse(all(ord(char) < 128 for char in csvDesc.get_column_names()[-2]))
def test2_prepareForVesperKrig_LowDensity(self):
file_csv = os.path.realpath(this_dir + '/area2_lowdensity_points.csv')
grid_file = os.path.realpath(this_dir +
"/rasters/area2_5m_blockgrid_v.txt")
data_col = 'Hand_Sample'
csv_desc = CsvDescribe(file_csv)
ctrl_para = VesperControl()
ctrl_para.update({
'jpntkrg': 1,
'jlockrg': 0,
'minpts': csv_desc.row_count - 2,
'maxpts': csv_desc.row_count,
'jcomvar': 0,
'modtyp': 'Spherical',
'iwei': 'no_pairs/variance',
'CO': 92.71,
'C1': 277.9,
'A1': 116.0
})
file_bat, file_ctrl = prepare_for_vesper_krige(
csv_desc.open_pandas_dataframe(),
data_col,
grid_file,
TEMPDIR,
control_textfile='test_low_control.txt',
control_options=ctrl_para,
coord_columns=[],
epsg=28354)
if os.path.exists(kriging_ops.vesper_exe):
self.assertTrue(
os.path.exists(os.path.join(TEMPDIR, 'Vesper/Do_Vesper.bat')))
else:
self.assertEqual('', file_bat)
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv')))
df_csv = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv'))
x_column, y_column = predictCoordinateColumnNames(df_csv.columns)
self.assertEqual('EASTING', x_column.upper())
self.assertEqual('NORTHING', y_column.upper())
with open(file_ctrl) as r_file:
data = r_file.read()
self.assertIn("outfil='test_low_kriged.txt'", data)
self.assertIn("outdir=''", data)
self.assertIn("jpntkrg=1", data)
self.assertIn("jlockrg=0", data)
self.assertIn("minpts=198", data)
self.assertIn("maxpts=200", data)
self.assertIn("modtyp=1", data)
self.assertIn("jcomvar=0", data)
self.assertIn("iwei=3", data)
self.assertIn("CO=92.71", data)
del data
src_df = pd.read_csv(
os.path.realpath(this_dir + '/VESPER/low_vesperdata.csv'))
test_df = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv'))
pd.testing.assert_frame_equal(src_df, test_df)
with open(os.path.realpath(this_dir + '/VESPER/low_control.txt')) as src_file,\
open(os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt')) as test_file:
self.assertEqual(src_file.readlines()[11:],
test_file.readlines()[11:])