def test_predictCoordinateFieldnames(self): self.assertEqual( predictCoordinateColumnNames(['longitude', 'latitude', 'area', 'weight', 'tonnes/ha', 'Air Temp(degC)']), ['longitude', 'latitude']) self.assertEqual(predictCoordinateColumnNames(['northing', 'name', 'easting', ' test type']), ['easting', 'northing']) self.assertEqual(predictCoordinateColumnNames(['name', 'lon_dms', 'lat_dms', ' test type']), ['lon_dms', 'lat_dms']) self.assertEqual(predictCoordinateColumnNames(['row', 'column', 'name', ' test type']), [None, None]) self.assertEqual( predictCoordinateColumnNames(['northing', 'easting', 'name', ' test type', 'longitude', 'latitude']), ['longitude', 'latitude'])
def test_from_ISO_8859_1_csv(self): file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv") descCSV = CsvDescribe(file) self.assertEqual(predictCoordinateColumnNames(descCSV.get_column_names()), ['Longitude', 'Latitude']) df = descCSV.open_pandas_dataframe() self.assertListEqual(df.columns.to_list(), descCSV.get_column_names())
def test06_prepareForVesperKrig(self): csv_desc = CsvDescribe(fileTrimmed) df_csv = csv_desc.open_pandas_dataframe() global file_control sub_file = os.path.splitext(os.path.basename(file_csv))[0] file_control = sub_file + '_control_' + data_col + '.txt' vc = VesperControl() vc.update(xside=30, yside=30) if not os.path.exists(file_control): bat_file, file_control = prepare_for_vesper_krige(df_csv, data_col, file_block_txt, TmpDir, control_textfile=file_control, coord_columns=[], epsg=epsg, control_options=vc) self.assertTrue(os.path.exists(bat_file)) self.assertTrue(os.path.exists(file_control)) self.assertTrue(os.path.exists(os.path.join(TmpDir, r'Vesper', sub_file + '_vesperdata_' + data_col + '.csv'))) df_csv = pd.read_csv(os.path.join(TmpDir, r'Vesper', sub_file + '_vesperdata_' + data_col + '.csv')) x_column, y_column = predictCoordinateColumnNames(df_csv.columns) self.assertEqual(x_column.upper(), 'EASTING') self.assertEqual(y_column.upper(), 'NORTHING') print('Running Vesper, Please wait....') run_vesper(file_control)
def test01_csvDescribe_ASCII(self): csv_desc = CsvDescribe(file_csv) self.assertEqual(csv_desc.file_encoding, 'ascii') self.assertEqual(csv_desc.row_count, 13756) self.assertEqual(csv_desc.column_count, 24) self.assertEqual(predictCoordinateColumnNames(csv_desc.get_column_names()), ['Lon', 'Lat']) self.assertTrue(csv_desc.has_column_header)
def test_csvfile_UTF8(self): csvDesc = CsvDescribe( os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")) self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1') self.assertEqual(csvDesc.row_count, 1543) self.assertEqual(csvDesc.column_count, 18) self.assertEqual( predictCoordinateColumnNames(csvDesc.get_column_names()), ['Longitude', 'Latitude']) self.assertTrue(csvDesc.has_column_header)
def loadTablePreview(self): # # build a dictionary of args dependent on form selections readArgs = {} readArgs['encoding'] = self.source_file['encoding'] if not self.chkHasHeader.isChecked(): readArgs['header'] = None readArgs['prefix'] = 'col_' else: if self.spnHeaderRowStart.value() > 1: readArgs['header'] = list(range(self.spnHeaderRowStart.value() - 1, self.spnHeaderRowEnd.value())) if self.spnIgnoreRows.value() > 0: readArgs['skiprows'] = range(self.spnHeaderRowEnd.value(), self.spnHeaderRowEnd.value() + self.spnIgnoreRows.value()) # if the table has already been read once before then we have the field types and # dont need to load the entire table which will speed things up if len(self.source_file['field_types']) > 0: readArgs['nrows'] = self.spnPreviewRowCount.value() readArgs['dtype'] = self.source_file['field_types'] if self.source_file['dialect'].delimiter == ',': df = pd.read_csv(self.source_file['file'], **readArgs) else: readArgs['sep'] = self.source_file['dialect'].delimiter df = pd.read_table(self.source_file['file'], **readArgs) if len(self.source_file['field_types']) == 0: self.source_file['field_types'] = df.dtypes.to_dict() df = df[:self.spnPreviewRowCount.value()] model = PandasModel(df) self.tvwSample.setModel(model) # #https://centaurialpha.github.io/resize-qheaderview-to-contents-and-interactive self.tvwSample.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents) # get numeric fields #df.select_dtypes(include=np.number).columns.tolist() #df.select_dtypes('number').columns coord_cols = predictCoordinateColumnNames(df.columns.tolist()) for i,obj in enumerate([self.cboXField,self.cboYField]): obj.clear() obj.addItems([' '] + df.select_dtypes('number').columns.tolist()) index = obj.findText(coord_cols[i], QtCore.Qt.MatchFixedString) if index >= 0: obj.setCurrentIndex(index)
def test1_prepareForVesperKrig_filesExist(self): gdfPoints, gdfPtsCrs = convert.convert_csv_to_points( file, coord_columns_epsg=4326, out_epsg=28354) outGDF, outCRS = clean_trim_points(gdfPoints, gdfPtsCrs, data_col, fileSubName + '_trimmed.csv', poly, thin_dist_m=2.5) self.assertTrue(os.path.exists(fileSubName + '_trimmed.csv')) self.assertEqual(outGDF.crs, {'init': 'epsg:28354', 'no_defs': True}) self.assertEqual(len(outGDF), 554) processing.block_grid(in_shapefilename=poly, pixel_size=5, out_rasterfilename=block_tif, out_vesperfilename=fileSubName + '_block_v.txt', snap=True, overwrite=True) global file_ctrl file_bat, file_ctrl = prepare_for_vesper_krige( outGDF, data_col, fileSubName + '_block_v.txt', TmpDir, control_textfile=os.path.basename(fileSubName) + '_control.txt', block_size=30, coord_columns=[], epsg=28354) self.assertTrue( os.path.exists(os.path.join(TmpDir, r'Vesper\Do_Vesper.bat'))) self.assertTrue( os.path.exists( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_control.txt'))) self.assertTrue( os.path.exists( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_vesperdata.csv'))) dfCSV = pd.read_csv( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_vesperdata.csv')) x_column, y_column = predictCoordinateColumnNames(dfCSV.columns) self.assertEqual(x_column.upper(), 'EASTING') self.assertEqual(y_column.upper(), 'NORTHING')
def test_csvfile_UTF8(self): csvDesc = CsvDescribe(os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")) # chardet seems to detect this file as ISO-8869-9, which is described # as "Largely the same as ISO/IEC 8859-1, replacing the rarely used # Icelandic letters with Turkish ones." #self.assertEqual(csvDesc.file_encoding, 'ISO-8859-1') self.assertEqual(csvDesc.file_encoding, 'ISO-8859-9') self.assertEqual(csvDesc.row_count, 1543) self.assertEqual(csvDesc.column_count, 18) self.assertEqual(predictCoordinateColumnNames(csvDesc.get_column_names()), ['Longitude', 'Latitude']) self.assertTrue(csvDesc.has_column_header) self.assertEqual(csvDesc.get_column_names()[-1], csvDesc.get_alias_column_names()[-1]) self.assertNotEqual(csvDesc.get_column_names()[-2], csvDesc.get_alias_column_names()[-2]) self.assertEqual(u'Crop Flw(V)(m\xb3/s)', csvDesc.get_column_names()[-2]) self.assertEqual('CropFlw(V)(m3/s)', csvDesc.get_alias_column_names()[-2]) #check to see if unicode characters exist True if all ascii, false if not self.assertTrue(all(ord(char) < 128 for char in csvDesc.get_alias_column_names()[-2])) self.assertFalse(all(ord(char) < 128 for char in csvDesc.get_column_names()[-2]))
def validate_csv_grid_files(self, csv_file, grid_file, show_msgbox=True): """ validate the csv and grid files and check for overlap assuming that they are of the same coordinate system if True then message will be blank else a message will be generated """ overlaps = False if csv_file != '': if not os.path.exists(csv_file): return False, 'CSV file does not exist' else: try: df_csv = pd.read_csv(csv_file) csvX, csvY = predictCoordinateColumnNames(df_csv.columns) csv_bbox = box(df_csv[csvX].min(), df_csv[csvY].min(), df_csv[csvX].max(), df_csv[csvY].max()) except Exception as err: self.lblInCSVFile.setStyleSheet('color:red') self.lneInCSVFile.setStyleSheet('color:red') return False, 'Invalid CSV file' if grid_file != '': if not os.path.exists(grid_file): return False, 'Grid file does not exist' else: try: df_grid = pd.read_table(grid_file, names=['X', 'Y'], delimiter=' ', skipinitialspace=True) grid_bbox = box(df_grid['X'].min(), df_grid['Y'].min(), df_grid['X'].max(), df_grid['Y'].max()) except Exception as err: self.lblInGridFile.setStyleSheet('color:red') self.lneInGridFile.setStyleSheet('color:red') return False, 'Invalid VESPER grid file' # only continue if both inputs aren't blank if csv_file == '' or grid_file == '': # if one or the other is blank keep validate as true or it wont write to the GUI return True, None if csv_file == grid_file: return False, "VESPER grid file and CSV file cannot be the same file" # now we can check for overlap if self.in_qgscrs: epsg = self.in_qgscrs.authid() else: epsg = '' overlaps = check_for_overlap(csv_bbox.to_wkt(), grid_bbox.to_wkt(), epsg, epsg) if not overlaps: message = 'There is no overlap between the VESPER Grid file and the CSV file.\n' \ 'Please check input files and coordinate systems' if show_msgbox: QMessageBox.warning(self, 'No Overlap', message) self.lblInGridFile.setStyleSheet('color:red') self.lneInGridFile.setStyleSheet('color:red') self.lblInCSVFile.setStyleSheet('color:red') self.lneInCSVFile.setStyleSheet('color:red') else: message = None self.lblInGridFile.setStyleSheet('color:black') self.lneInGridFile.setStyleSheet('color:black') self.lblInCSVFile.setStyleSheet('color:black') self.lneInCSVFile.setStyleSheet('color:black') return overlaps, message
def on_cmdInCSVFile_clicked(self): self.lneInCSVFile.clear() self.messageBar.clearWidgets() self.cboMethod.setCurrentIndex(0) self.dfCSV = None inFolder = read_setting(PLUGIN_NAME + "/" + self.toolKey + "/LastCSVFolder") if inFolder is None or not os.path.exists(inFolder): inFolder = read_setting(PLUGIN_NAME + '/BASE_IN_FOLDER') s = QtGui.QFileDialog.getOpenFileName(self, caption=self.tr("Select a CSV file to krige"), directory=inFolder, filter='{} (*.csv);;{} (*.*);;'.format( self.tr("Comma delimited files"), self.tr("All Files")) ) self.cleanMessageBars(self) self.lneInCSVFile.clear() # validate files first overlaps, message = self.validate_csv_grid_files(s, self.lneInGridFile.text()) if not overlaps or message is not None: self.lblInCSVFile.setStyleSheet('color:red') self.lneInCSVFile.setStyleSheet('color:red') self.send_to_messagebar(message, level=QgsMessageBar.CRITICAL, duration=0, addToLog=True, showLogPanel=True, exc_info=sys.exc_info()) return s = os.path.normpath(s) self.lblInCSVFile.setStyleSheet('color:black') self.lneInCSVFile.setStyleSheet('color:black') self.lneInCSVFile.setText(s) descCSV = describe.CsvDescribe(s) self.dfCSV = descCSV.open_pandas_dataframe(nrows=150) if len(self.dfCSV) <= 100: self.lneMinPoint.clear() QMessageBox.warning(self, 'Cannot Krige', 'Kriging is not advised for less ' 'than 100 points') self.lneMinPoint.clear() self.cboKrigColumn.clear() coord_cols = predictCoordinateColumnNames(self.dfCSV.columns) epsgcols = [col for col in self.dfCSV.columns if 'EPSG' in col.upper()] field_names = list(self.dfCSV.drop(coord_cols + epsgcols, axis=1) .select_dtypes(include=[np.number]).columns.values) self.cboKrigColumn.addItems([''] + field_names) # To set a coordinate system for vesper2raster, try and get it from # a column in the data. epsg = 0 if len(epsgcols) > 0: for col in epsgcols: if self.dfCSV.iloc[0][col] > 0: epsg = int(self.dfCSV.iloc[0][col]) break if epsg > 0: self.in_qgscrs = QgsCoordinateReferenceSystem("EPSG:{}".format(epsg)) self.lblInCRS.setText('{} - {}'.format(self.in_qgscrs.description(), self.in_qgscrs.authid())) else: self.lblInCRS.setText('Unspecified') write_setting(PLUGIN_NAME + "/" + self.toolKey + "/LastCSVFolder", os.path.dirname(s)) del descCSV self.updateCtrlFileName()
def test2_prepareForVesperKrig_LowDensity(self): file_csv = os.path.realpath(this_dir + '/area2_lowdensity_points.csv') grid_file = os.path.realpath(this_dir + "/rasters/area2_5m_blockgrid_v.txt") data_col = 'Hand_Sample' csv_desc = CsvDescribe(file_csv) ctrl_para = VesperControl() ctrl_para.update({ 'jpntkrg': 1, 'jlockrg': 0, 'minpts': csv_desc.row_count - 2, 'maxpts': csv_desc.row_count, 'jcomvar': 0, 'modtyp': 'Spherical', 'iwei': 'no_pairs/variance', 'CO': 92.71, 'C1': 277.9, 'A1': 116.0 }) file_bat, file_ctrl = prepare_for_vesper_krige( csv_desc.open_pandas_dataframe(), data_col, grid_file, TEMPDIR, control_textfile='test_low_control.txt', control_options=ctrl_para, coord_columns=[], epsg=28354) if os.path.exists(kriging_ops.vesper_exe): self.assertTrue( os.path.exists(os.path.join(TEMPDIR, 'Vesper/Do_Vesper.bat'))) else: self.assertEqual('', file_bat) self.assertTrue( os.path.exists( os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt'))) self.assertTrue( os.path.exists( os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv'))) df_csv = pd.read_csv( os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv')) x_column, y_column = predictCoordinateColumnNames(df_csv.columns) self.assertEqual('EASTING', x_column.upper()) self.assertEqual('NORTHING', y_column.upper()) with open(file_ctrl) as r_file: data = r_file.read() self.assertIn("outfil='test_low_kriged.txt'", data) self.assertIn("outdir=''", data) self.assertIn("jpntkrg=1", data) self.assertIn("jlockrg=0", data) self.assertIn("minpts=198", data) self.assertIn("maxpts=200", data) self.assertIn("modtyp=1", data) self.assertIn("jcomvar=0", data) self.assertIn("iwei=3", data) self.assertIn("CO=92.71", data) del data src_df = pd.read_csv( os.path.realpath(this_dir + '/VESPER/low_vesperdata.csv')) test_df = pd.read_csv( os.path.join(TEMPDIR, 'Vesper', 'test_low_vesperdata.csv')) pd.testing.assert_frame_equal(src_df, test_df) with open(os.path.realpath(this_dir + '/VESPER/low_control.txt')) as src_file,\ open(os.path.join(TEMPDIR, 'Vesper', 'test_low_control.txt')) as test_file: self.assertEqual(src_file.readlines()[11:], test_file.readlines()[11:])