def test_predictCoordinateFieldnames(self):
self.assertEqual(
predictCoordinateColumnNames(['longitude', 'latitude', 'area', 'weight', 'tonnes/ha', 'Air Temp(degC)']),
['longitude', 'latitude'])
self.assertEqual(predictCoordinateColumnNames(['northing', 'name', 'easting', ' test type']),
['easting', 'northing'])
self.assertEqual(predictCoordinateColumnNames(['name', 'lon_dms', 'lat_dms', ' test type']),
['lon_dms', 'lat_dms'])
self.assertEqual(predictCoordinateColumnNames(['row', 'column', 'name', ' test type']), [None, None])
self.assertEqual(
predictCoordinateColumnNames(['northing', 'easting', 'name', ' test type', 'longitude', 'latitude']),
['longitude', 'latitude'])
def test_from_ISO_8859_1_csv(self):
file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")
descCSV = CsvDescribe(file)
self.assertEqual(predictCoordinateColumnNames(descCSV.get_column_names()), ['Longitude', 'Latitude'])
df = descCSV.open_pandas_dataframe()
self.assertListEqual(df.columns.to_list(), descCSV.get_column_names())
def test06_prepareForVesperKrig(self):
csv_desc = CsvDescribe(fileTrimmed)
df_csv = csv_desc.open_pandas_dataframe()
global file_control
sub_file = os.path.splitext(os.path.basename(file_csv))[0]
file_control = sub_file + '_control_' + data_col + '.txt'
vc = VesperControl()
vc.update(xside=30, yside=30)
if not os.path.exists(file_control):
bat_file, file_control = prepare_for_vesper_krige(df_csv, data_col, file_block_txt,
TmpDir,
control_textfile=file_control,
coord_columns=[], epsg=epsg,
control_options=vc)
self.assertTrue(os.path.exists(bat_file))
self.assertTrue(os.path.exists(file_control))
self.assertTrue(os.path.exists(os.path.join(TmpDir, r'Vesper',
sub_file + '_vesperdata_' + data_col + '.csv')))
df_csv = pd.read_csv(os.path.join(TmpDir, r'Vesper',
sub_file + '_vesperdata_' + data_col + '.csv'))
x_column, y_column = predictCoordinateColumnNames(df_csv.columns)
self.assertEqual(x_column.upper(), 'EASTING')
self.assertEqual(y_column.upper(), 'NORTHING')
print('Running Vesper, Please wait....')
run_vesper(file_control)
def test01_csvDescribe_ASCII(self):
csv_desc = CsvDescribe(file_csv)
self.assertEqual(csv_desc.file_encoding, 'ascii')
self.assertEqual(csv_desc.row_count, 13756)
self.assertEqual(csv_desc.column_count, 24)
self.assertEqual(predictCoordinateColumnNames(csv_desc.get_column_names()), ['Lon', 'Lat'])
self.assertTrue(csv_desc.has_column_header)
def test_csvfile_UTF8(self):
csvDesc = CsvDescribe(
os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv"))
self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1')
self.assertEqual(csvDesc.row_count, 1543)
self.assertEqual(csvDesc.column_count, 18)
self.assertEqual(
predictCoordinateColumnNames(csvDesc.get_column_names()),
['Longitude', 'Latitude'])
self.assertTrue(csvDesc.has_column_header)
def loadTablePreview(self):
# # build a dictionary of args dependent on form selections
readArgs = {}
readArgs['encoding'] = self.source_file['encoding']
if not self.chkHasHeader.isChecked():
readArgs['header'] = None
readArgs['prefix'] = 'col_'
else:
if self.spnHeaderRowStart.value() > 1:
readArgs['header'] = list(range(self.spnHeaderRowStart.value() - 1,
self.spnHeaderRowEnd.value()))
if self.spnIgnoreRows.value() > 0:
readArgs['skiprows'] = range(self.spnHeaderRowEnd.value(),
self.spnHeaderRowEnd.value() + self.spnIgnoreRows.value())
# if the table has already been read once before then we have the field types and
# dont need to load the entire table which will speed things up
if len(self.source_file['field_types']) > 0:
readArgs['nrows'] = self.spnPreviewRowCount.value()
readArgs['dtype'] = self.source_file['field_types']
if self.source_file['dialect'].delimiter == ',':
df = pd.read_csv(self.source_file['file'], **readArgs)
else:
readArgs['sep'] = self.source_file['dialect'].delimiter
df = pd.read_table(self.source_file['file'], **readArgs)
if len(self.source_file['field_types']) == 0:
self.source_file['field_types'] = df.dtypes.to_dict()
df = df[:self.spnPreviewRowCount.value()]
model = PandasModel(df)
self.tvwSample.setModel(model)
# #https://centaurialpha.github.io/resize-qheaderview-to-contents-and-interactive
self.tvwSample.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
# get numeric fields
#df.select_dtypes(include=np.number).columns.tolist()
#df.select_dtypes('number').columns
coord_cols = predictCoordinateColumnNames(df.columns.tolist())
for i,obj in enumerate([self.cboXField,self.cboYField]):
obj.clear()
obj.addItems([' '] + df.select_dtypes('number').columns.tolist())
index = obj.findText(coord_cols[i], QtCore.Qt.MatchFixedString)
if index >= 0:
obj.setCurrentIndex(index)
def test1_prepareForVesperKrig_filesExist(self):
gdfPoints, gdfPtsCrs = convert.convert_csv_to_points(
file, coord_columns_epsg=4326, out_epsg=28354)
outGDF, outCRS = clean_trim_points(gdfPoints,
gdfPtsCrs,
data_col,
fileSubName + '_trimmed.csv',
poly,
thin_dist_m=2.5)
self.assertTrue(os.path.exists(fileSubName + '_trimmed.csv'))
self.assertEqual(outGDF.crs, {'init': 'epsg:28354', 'no_defs': True})
self.assertEqual(len(outGDF), 554)
processing.block_grid(in_shapefilename=poly,
pixel_size=5,
out_rasterfilename=block_tif,
out_vesperfilename=fileSubName + '_block_v.txt',
snap=True,
overwrite=True)
global file_ctrl
file_bat, file_ctrl = prepare_for_vesper_krige(
outGDF,
data_col,
fileSubName + '_block_v.txt',
TmpDir,
control_textfile=os.path.basename(fileSubName) + '_control.txt',
block_size=30,
coord_columns=[],
epsg=28354)
self.assertTrue(
os.path.exists(os.path.join(TmpDir, r'Vesper\Do_Vesper.bat')))
self.assertTrue(
os.path.exists(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) + '_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) +
'_vesperdata.csv')))
dfCSV = pd.read_csv(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) + '_vesperdata.csv'))
x_column, y_column = predictCoordinateColumnNames(dfCSV.columns)
self.assertEqual(x_column.upper(), 'EASTING')
self.assertEqual(y_column.upper(), 'NORTHING')
def test_csvfile_UTF8(self):
csvDesc = CsvDescribe(os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv"))
# chardet seems to detect this file as ISO-8869-9, which is described
# as "Largely the same as ISO/IEC 8859-1, replacing the rarely used
# Icelandic letters with Turkish ones."
#self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1')
self.assertEqual(csvDesc.file_encoding, 'ISO-8859-9')
self.assertEqual(csvDesc.row_count, 1543)
self.assertEqual(csvDesc.column_count, 18)
self.assertEqual(predictCoordinateColumnNames(csvDesc.get_column_names()), ['Longitude', 'Latitude'])
self.assertTrue(csvDesc.has_column_header)
self.assertEqual(csvDesc.get_column_names()[-1], csvDesc.get_alias_column_names()[-1])
self.assertNotEqual(csvDesc.get_column_names()[-2], csvDesc.get_alias_column_names()[-2])
self.assertEqual(u'Crop Flw(V)(m\xb3/s)', csvDesc.get_column_names()[-2])
self.assertEqual('CropFlw(V)(m3/s)', csvDesc.get_alias_column_names()[-2])
#check to see if unicode characters exist True if all ascii, false if not
self.assertTrue(all(ord(char) < 128 for char in csvDesc.get_alias_column_names()[-2]))
self.assertFalse(all(ord(char) < 128 for char in csvDesc.get_column_names()[-2]))
def validate_csv_grid_files(self, csv_file, grid_file, show_msgbox=True):
""" validate the csv and grid files and check for overlap assuming that they are
of the same coordinate system if True then message will be blank else a message
will be generated
"""
overlaps = False
if csv_file != '':
if not os.path.exists(csv_file):
return False, 'CSV file does not exist'
else:
try:
df_csv = pd.read_csv(csv_file)
csvX, csvY = predictCoordinateColumnNames(df_csv.columns)
csv_bbox = box(df_csv[csvX].min(), df_csv[csvY].min(),
df_csv[csvX].max(), df_csv[csvY].max())
except Exception as err:
self.lblInCSVFile.setStyleSheet('color:red')
self.lneInCSVFile.setStyleSheet('color:red')
return False, 'Invalid CSV file'
if grid_file != '':
if not os.path.exists(grid_file):
return False, 'Grid file does not exist'
else:
try:
df_grid = pd.read_table(grid_file, names=['X', 'Y'],
delimiter=' ', skipinitialspace=True)
grid_bbox = box(df_grid['X'].min(), df_grid['Y'].min(),
df_grid['X'].max(), df_grid['Y'].max())
except Exception as err:
self.lblInGridFile.setStyleSheet('color:red')
self.lneInGridFile.setStyleSheet('color:red')
return False, 'Invalid VESPER grid file'
# only continue if both inputs aren't blank
if csv_file == '' or grid_file == '':
# if one or the other is blank keep validate as true or it wont write to the GUI
return True, None
if csv_file == grid_file:
return False, "VESPER grid file and CSV file cannot be the same file"
# now we can check for overlap
if self.in_qgscrs:
epsg = self.in_qgscrs.authid()
else:
epsg = ''
overlaps = check_for_overlap(csv_bbox.to_wkt(), grid_bbox.to_wkt(), epsg, epsg)
if not overlaps:
message = 'There is no overlap between the VESPER Grid file and the CSV file.\n' \
'Please check input files and coordinate systems'
if show_msgbox:
QMessageBox.warning(self, 'No Overlap', message)
self.lblInGridFile.setStyleSheet('color:red')
self.lneInGridFile.setStyleSheet('color:red')
self.lblInCSVFile.setStyleSheet('color:red')
self.lneInCSVFile.setStyleSheet('color:red')
else:
message = None
self.lblInGridFile.setStyleSheet('color:black')
self.lneInGridFile.setStyleSheet('color:black')
self.lblInCSVFile.setStyleSheet('color:black')
self.lneInCSVFile.setStyleSheet('color:black')
return overlaps, message
def on_cmdInCSVFile_clicked(self):
self.lneInCSVFile.clear()
self.messageBar.clearWidgets()
self.cboMethod.setCurrentIndex(0)
self.dfCSV = None
inFolder = read_setting(PLUGIN_NAME + "/" + self.toolKey + "/LastCSVFolder")
if inFolder is None or not os.path.exists(inFolder):
inFolder = read_setting(PLUGIN_NAME + '/BASE_IN_FOLDER')
s = QtGui.QFileDialog.getOpenFileName(self,
caption=self.tr("Select a CSV file to krige"),
directory=inFolder,
filter='{} (*.csv);;{} (*.*);;'.format(
self.tr("Comma delimited files"),
self.tr("All Files"))
)
self.cleanMessageBars(self)
self.lneInCSVFile.clear()
# validate files first
overlaps, message = self.validate_csv_grid_files(s, self.lneInGridFile.text())
if not overlaps or message is not None:
self.lblInCSVFile.setStyleSheet('color:red')
self.lneInCSVFile.setStyleSheet('color:red')
self.send_to_messagebar(message,
level=QgsMessageBar.CRITICAL,
duration=0, addToLog=True, showLogPanel=True,
exc_info=sys.exc_info())
return
s = os.path.normpath(s)
self.lblInCSVFile.setStyleSheet('color:black')
self.lneInCSVFile.setStyleSheet('color:black')
self.lneInCSVFile.setText(s)
descCSV = describe.CsvDescribe(s)
self.dfCSV = descCSV.open_pandas_dataframe(nrows=150)
if len(self.dfCSV) <= 100:
self.lneMinPoint.clear()
QMessageBox.warning(self, 'Cannot Krige', 'Kriging is not advised for less '
'than 100 points')
self.lneMinPoint.clear()
self.cboKrigColumn.clear()
coord_cols = predictCoordinateColumnNames(self.dfCSV.columns)
epsgcols = [col for col in self.dfCSV.columns if 'EPSG' in col.upper()]
field_names = list(self.dfCSV.drop(coord_cols + epsgcols, axis=1)
.select_dtypes(include=[np.number]).columns.values)
self.cboKrigColumn.addItems([''] + field_names)
# To set a coordinate system for vesper2raster, try and get it from
# a column in the data.
epsg = 0
if len(epsgcols) > 0:
for col in epsgcols:
if self.dfCSV.iloc[0][col] > 0:
epsg = int(self.dfCSV.iloc[0][col])
break
if epsg > 0:
self.in_qgscrs = QgsCoordinateReferenceSystem("EPSG:{}".format(epsg))
self.lblInCRS.setText('{} - {}'.format(self.in_qgscrs.description(),
self.in_qgscrs.authid()))
else:
self.lblInCRS.setText('Unspecified')
write_setting(PLUGIN_NAME + "/" + self.toolKey +
"/LastCSVFolder", os.path.dirname(s))
del descCSV
self.updateCtrlFileName()
def test2_prepareForVesperKrig_LowDensity(self):
file_csv = os.path.realpath(this_dir + '/area2_lowdensity_points.csv')
grid_file = os.path.realpath(this_dir +
"/rasters/area2_5m_blockgrid_v.txt")
data_col = 'Hand_Sample'
csv_desc = CsvDescribe(file_csv)
ctrl_para = VesperControl()
ctrl_para.update({
'jpntkrg': 1,
'jlockrg': 0,
'minpts': csv_desc.row_count - 2,
'maxpts': csv_desc.row_count,
'jcomvar': 0,
'modtyp': 'Spherical',
'iwei': 'no_pairs/variance',
'CO': 92.71,
'C1': 277.9,
'A1': 116.0
})
file_bat, file_ctrl = prepare_for_vesper_krige(
csv_desc.open_pandas_dataframe(),
data_col,
grid_file,
TEMPDIR,
control_textfile='test_low_control.txt',
control_options=ctrl_para,
coord_columns=[],
epsg=28354)
if os.path.exists(kriging_ops.vesper_exe):
self.assertTrue(
os.path.exists(os.path.join(TEMPDIR, 'Vesper/Do_Vesper.bat')))
else:
self.assertEqual('', file_bat)
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv')))
df_csv = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv'))
x_column, y_column = predictCoordinateColumnNames(df_csv.columns)
self.assertEqual('EASTING', x_column.upper())
self.assertEqual('NORTHING', y_column.upper())
with open(file_ctrl) as r_file:
data = r_file.read()
self.assertIn("outfil='test_low_kriged.txt'", data)
self.assertIn("outdir=''", data)
self.assertIn("jpntkrg=1", data)
self.assertIn("jlockrg=0", data)
self.assertIn("minpts=198", data)
self.assertIn("maxpts=200", data)
self.assertIn("modtyp=1", data)
self.assertIn("jcomvar=0", data)
self.assertIn("iwei=3", data)
self.assertIn("CO=92.71", data)
del data
src_df = pd.read_csv(
os.path.realpath(this_dir + '/VESPER/low_vesperdata.csv'))
test_df = pd.read_csv(
os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv'))
pd.testing.assert_frame_equal(src_df, test_df)
with open(os.path.realpath(this_dir + '/VESPER/low_control.txt')) as src_file,\
open(os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt')) as test_file:
self.assertEqual(src_file.readlines()[11:],
test_file.readlines()[11:])