def test05_cleanTrimPoints(self): global fileTrimmed, data_col fileTrimmed = os.path.join(TmpDir, os.path.splitext( os.path.basename(file_csv))[0] + '_normtrimmed.csv') file_shp = os.path.join(TmpDir, os.path.splitext( os.path.basename(file_csv))[0] + '_normtrimmed.shp') file_removed = os.path.join(TmpDir, os.path.splitext( os.path.basename(file_csv))[0] + '_remove.shp') data_col = r'Yield' gdf_points, gdfpts_crs = convert.convert_csv_to_points(file_csv, coord_columns_epsg=4326, out_epsg=epsg) gdf_out, crs_out = clean_trim_points(gdf_points, gdfpts_crs, data_col, fileTrimmed, out_keep_shapefile=file_shp, out_removed_shapefile=file_removed, boundary_polyfile=fileBox, thin_dist_m=2.5) self.assertTrue(os.path.exists(fileTrimmed)) self.assertTrue(os.path.exists(file_shp)) self.assertTrue(os.path.exists(file_removed)) self.assertEqual(gdf_out.crs, crs.from_epsg(epsg)) self.assertEqual(len(gdf_out), 648) self.assertIn('nrm_' + data_col, gdf_out.columns) self.assertIn('Easting', gdf_out.columns) self.assertIn('Northing', gdf_out.columns) self.assertIn('EN_EPSG', gdf_out.columns)
def test_convert_csv_to_points_WGS84(self): file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv") out_epsg = 28354 out_file = os.path.join( TmpDir, os.path.basename(file).replace('.csv', '_{}.shp'.format(out_epsg))) gdfData, gdfCRS = convert_csv_to_points(file, out_file, coord_columns_epsg=4326, out_epsg=out_epsg) self.assertTrue(os.path.exists(out_file)) self.assertIsInstance(gdfData, GeoDataFrame) self.assertEqual(len(gdfData), 1543) self.assertEqual(list(set(gdfData.geom_type)), ['Point']) import numpy as np np.testing.assert_almost_equal( list(gdfData.total_bounds), [598868.6709, 6054050.0529, 599242.9049, 6054415.3845], 4) self.assertEqual(gdfCRS.epsg_number, out_epsg) self.assertEqual( gdfCRS.crs_wkt, 'PROJCS["GDA94 / MGA zone 54",GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4283"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",141],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",10000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","28354"]]' )
def test_cleanTrimPoints(self): in_csv = os.path.join(this_dir + "/data/area2_yield_ISO-8859-1.csv") in_poly = os.path.join(this_dir + "/data/area2_onebox_94mga54.shp") out_csv = os.path.join(TmpDir, os.path.basename(in_csv)) out_shp = os.path.join( TmpDir, os.path.basename(in_csv).replace('.csv', '.shp')) out_rm_shp = os.path.join( TmpDir, os.path.basename(in_csv).replace('.csv', '_remove.shp')) gdf_points, gdf_pts_crs = convert.convert_csv_to_points( in_csv, coord_columns_epsg=4326, out_epsg=28354) out_gdf, out_crs = clean_trim_points(gdf_points, gdf_pts_crs, 'Yld Mass(Dry)(tonne/ha)', out_csv, out_keep_shapefile=out_shp, out_removed_shapefile=out_rm_shp, boundary_polyfile=in_poly, thin_dist_m=2.5) self.assertIsInstance(out_gdf, GeoDataFrame) self.assertTrue(os.path.exists(out_csv)) self.assertTrue(gdf_pts_crs, out_crs) self.assertEqual(out_gdf.crs, {'init': 'epsg:28354', 'no_defs': True}) self.assertEqual(len(out_gdf), 554) self.assertIn('EN_EPSG', out_gdf.columns)
def test_createPolygonFromPointTrail(self): in_csv = os.path.join(this_dir + "/data/area2_yield_ISO-8859-1.csv") # out_ptsfile = os.path.join(TmpDir, os.path.splitext(os.path.basename(in_csv))[0] + '_points.shp') out_polyfile = os.path.join( TmpDir, os.path.splitext(os.path.basename(in_csv))[0] + '_poly.shp') gdf_points, gdf_pts_crs = convert.convert_csv_to_points( in_csv, None, coord_columns_epsg=4326, out_epsg=28354) create_polygon_from_point_trail(gdf_points, gdf_pts_crs, out_polyfile, thin_dist_m=2.5, aggregate_dist_m=25, buffer_dist_m=10, shrink_dist_m=3) self.assertTrue(os.path.exists(out_polyfile), True) vect_desc = VectorDescribe(out_polyfile) self.assertEqual(vect_desc.crs.epsg_number, 28354) self.assertFalse(vect_desc.is_mz_aware) self.assertEqual(vect_desc.geometry_type, 'Polygon')
def test1_prepareForVesperKrig_filesExist(self): gdfPoints, gdfPtsCrs = convert.convert_csv_to_points( file, coord_columns_epsg=4326, out_epsg=28354) outGDF, outCRS = clean_trim_points(gdfPoints, gdfPtsCrs, data_col, fileSubName + '_trimmed.csv', poly, thin_dist_m=2.5) self.assertTrue(os.path.exists(fileSubName + '_trimmed.csv')) self.assertEqual(outGDF.crs, {'init': 'epsg:28354', 'no_defs': True}) self.assertEqual(len(outGDF), 554) processing.block_grid(in_shapefilename=poly, pixel_size=5, out_rasterfilename=block_tif, out_vesperfilename=fileSubName + '_block_v.txt', snap=True, overwrite=True) global file_ctrl file_bat, file_ctrl = prepare_for_vesper_krige( outGDF, data_col, fileSubName + '_block_v.txt', TmpDir, control_textfile=os.path.basename(fileSubName) + '_control.txt', block_size=30, coord_columns=[], epsg=28354) self.assertTrue( os.path.exists(os.path.join(TmpDir, r'Vesper\Do_Vesper.bat'))) self.assertTrue( os.path.exists( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_control.txt'))) self.assertTrue( os.path.exists( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_vesperdata.csv'))) dfCSV = pd.read_csv( os.path.join(TmpDir, r'Vesper', os.path.basename(fileSubName) + '_vesperdata.csv')) x_column, y_column = predictCoordinateColumnNames(dfCSV.columns) self.assertEqual(x_column.upper(), 'EASTING') self.assertEqual(y_column.upper(), 'NORTHING')
def test_thinPointByDistance_mga54(self): file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv") out_epsg = 28354 out_file = os.path.join( TmpDir, os.path.basename(file).replace('.csv', '_{}.shp'.format(out_epsg))) gdf, gdfCRS = convert_csv_to_points(file, out_file, coord_columns_epsg=4326, out_epsg=out_epsg) result = thin_point_by_distance(gdf, gdfCRS, 2.5) self.assertEqual(len(result), len(gdf)) self.assertEqual(len(result[result['filter'].isnull()]), 1434) self.assertEqual(result.crs, gdf.crs)
def test_convert_csv_to_points_EastingNorthings(self): file = os.path.realpath(this_dir + "/data/area1_yield_ascii_wgs84.csv") epsg = 28354 out_file = os.path.join( TmpDir, os.path.basename(file).replace('.csv', '_{}.shp'.format(epsg))) gdfData, gdfCRS = convert_csv_to_points( file, out_file, coord_columns=['Easting', 'Northing'], coord_columns_epsg=epsg) self.assertIsInstance(gdfData, GeoDataFrame) self.assertEqual(len(gdfData), 13756) self.assertEqual(gdfCRS.epsg_number, epsg) self.assertTrue(os.path.exists(out_file))
def test02_createPolygonFromPointTrail(self): global filePoints, filePoly filePoints = os.path.join(TmpDir, os.path.splitext(os.path.basename(file_csv))[0] + '_points.shp') filePoly = os.path.join(TmpDir, os.path.splitext(os.path.basename(file_csv))[0] + '_poly.shp') if not os.path.exists(filePoly): gdf_points, gdf_pts_crs = convert.convert_csv_to_points(file_csv, filePoints, coord_columns_epsg=4326, out_epsg=epsg) create_polygon_from_point_trail(gdf_points, gdf_pts_crs, filePoly, thin_dist_m=2.5, aggregate_dist_m=25, buffer_dist_m=7, shrink_dist_m=3) self.assertTrue(os.path.exists(filePoly), True)
def accept(self, *args, **kwargs): if not self.validate(): return False try: # disable form via a frame, this will still allow interaction with the message bar self.stackedWidget.setDisabled(True) # clean gui and Qgis messagebars self.cleanMessageBars(True) # self.iface.messageBar().clearWidgets() # Change cursor to Wait cursor QApplication.setOverrideCursor(QtCore.Qt.WaitCursor) self.iface.mainWindow().statusBar().showMessage('Processing {}'.format(self.windowTitle())) self.send_to_messagebar("Please wait.. QGIS will be locked... See log panel for progress.", level=Qgis.Warning, duration=0, addToLog=False, core_QGIS=False, showLogPanel=True) gp_layer_name = '' LOGGER.info('{st}\nProcessing {}'.format(self.windowTitle(), st='*' * 50)) # Add settings to log settingsStr = 'Parameters:---------------------------------------' if self.optFile.isChecked(): settingsStr += '\n {:30}\t{}'.format('File:', self.lneInCSVFile.text()) settingsStr += '\n {:30}\t{}, {}'.format('Geometry Fields:', self.cboXField.currentText(), self.cboYField.currentText()) settingsStr += '\n {:30}\t{} - {}'.format('CSV Coordinate System:', self.qgsCRScsv.crs().authid(), self.qgsCRScsv.crs().description()) else: if self.chkUseSelected.isChecked(): settingsStr += '\n {:30}\t{} with {} selected features'.format('Layer:', self.mcboTargetLayer.currentLayer().name(), self.mcboTargetLayer.currentLayer().selectedFeatureCount()) else: settingsStr += '\n {:30}\t{}'.format('Layer:', self.mcboTargetLayer.currentLayer().name()) crs_units = QgsUnitTypes.toString(self.mCRSoutput.crs().mapUnits()) settingsStr += '\n {:30}\t{} {}'.format('Thinning Distance:', self.dsbThinDist.value(),crs_units) settingsStr += '\n {:30}\t{} {}'.format("Aggregate Distance:", self.dsbAggregateDist.value(),crs_units) settingsStr += '\n {:30}\t{} {}'.format("Buffer Distance:", self.dsbBufferDist.value(),crs_units) settingsStr += '\n {:30}\t{} {}'.format("Shrink Distance:", self.dsbShrinkDist.value(),crs_units) settingsStr += '\n {:30}\t{}'.format('Output Polygon Shapefile:', self.lneSavePolyFile.text()) if self.lneSavePointsFile.text() == '': settingsStr += '\n {:30}\t{}'.format('Saved Points Shapefile:', self.lneSavePointsFile.text()) settingsStr += '\n {:30}\t{} - {}\n\n'.format('Output Projected Coordinate System:', self.mCRSoutput.crs().authid(), self.mCRSoutput.crs().description()) LOGGER.info(settingsStr) stepTime = time.time() if self.optFile.isChecked(): in_epsg = int(self.qgsCRScsv.crs().authid().replace('EPSG:','')) in_crs = self.qgsCRScsv.crs() else: in_epsg =self.mcboTargetLayer.currentLayer().crs().authid().replace('EPSG:','') in_crs = self.mcboTargetLayer.currentLayer().crs() out_epsg = int(self.mCRSoutput.crs().authid().replace('EPSG:','')) filePoly = None gdfPoints = None filePoints = None if self.optFile.isChecked(): if self.DEBUG: filePoints = os.path.join(TEMPDIR, os.path.splitext(os.path.basename(self.lneSavePolyFile.text()))[0] + '_table2pts.shp') if os.path.splitext(self.lneInCSVFile.text())[-1] == '.csv': gdfPoints, gdfPtsCrs = convert.convert_csv_to_points(self.lneInCSVFile.text() , out_shapefilename=filePoints, coord_columns=[self.cboXField.currentText(), self.cboYField.currentText()], coord_columns_epsg=in_epsg) elif os.path.splitext(self.lneInCSVFile.text())[-1] in ['.xls', '.xlsx', '.ods']: xls_file = pd.ExcelFile(self.lneInCSVFile.text()) pdfxls = xls_file.parse(self.sheet(), skiprows=self.linesToIgnore() - 1) del xls_file gdfPoints, gdfPtsCrs = convert.add_point_geometry_to_dataframe(pdfxls, coord_columns=[ self.cboXField.currentText(), self.cboYField.currentText()], coord_columns_epsg=in_epsg) del pdfxls LOGGER.info('{:<30} {d:<15} {}'.format('Add Geometry to Table','', d=str(timedelta(seconds=time.time() - stepTime)))) stepTime = time.time() if filePoints is not None: describe.save_geopandas_tofile(gdfPoints, filePoints) #, file_encoding=self.file_encoding) if self.DISP_TEMP_LAYERS and filePoints is not None: addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0], group_layer_name='DEBUG', atTop=True) else: layerPts = self.mcboTargetLayer.currentLayer() if layerPts.providerType() == 'delimitedtext' or \ os.path.splitext(get_layer_source(layerPts))[-1] == '.vrt' or \ self.chkUseSelected.isChecked() or self.optFile.isChecked(): filePoints = os.path.join(TEMPDIR, "{}_points.shp".format(layerPts.name())) if self.chkUseSelected.isChecked(): filePoints = os.path.join(TEMPDIR, "{}_selected_points.shp".format(layerPts.name())) if os.path.exists(filePoints): removeFileFromQGIS(filePoints) ptsLayer = copyLayerToMemory(layerPts, layerPts.name() + "_memory", bAddUFI=True, bOnlySelectedFeat=self.chkUseSelected.isChecked()) _writer = QgsVectorFileWriter.writeAsVectorFormat(ptsLayer, filePoints, "utf-8", self.mCRSoutput.crs(), driverName="ESRI Shapefile") LOGGER.info('{:<30} {d:<15} {}'.format('Save layer/selection to file',filePoints, d=str(timedelta(seconds=time.time() - stepTime) ))) stepTime = time.time() del ptsLayer if self.DISP_TEMP_LAYERS: addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0], group_layer_name='DEBUG', atTop=True) else: filePoints = get_layer_source(layerPts) if gdfPoints is None: ptsDesc = describe.VectorDescribe(filePoints) gdfPtsCrs = ptsDesc.crs gdfPoints = ptsDesc.open_geo_dataframe() if in_crs.authid() != self.mCRSoutput.crs().authid(): gdfPoints = gdfPoints.to_crs(epsg=out_epsg) gdfPtsCrs = pyprecag_crs.crs() gdfPtsCrs.getFromEPSG(out_epsg) LOGGER.info('{:<30} {d:<15} {} to {}'.format('Reproject points', in_crs.authid(), self.mCRSoutput.crs().authid(), d=str(timedelta(seconds=time.time() - stepTime)))) if self.DEBUG: filePoints = os.path.join(TEMPDIR, os.path.basename(self.lneSavePolyFile.text().replace('.csv', '_ptsprj.shp'))) removeFileFromQGIS(filePoints) describe.save_geopandas_tofile(gdfPoints, filePoints) if self.DISP_TEMP_LAYERS: if self.DEBUG: addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0], group_layer_name='DEBUG', atTop=True) else: addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0], atTop=True) stepTime = time.time() result = processing.create_polygon_from_point_trail(gdfPoints, gdfPtsCrs, out_filename=self.lneSavePolyFile.text(), thin_dist_m=self.dsbThinDist.value(), aggregate_dist_m=self.dsbAggregateDist.value(), buffer_dist_m=self.dsbBufferDist.value(), shrink_dist_m=self.dsbShrinkDist.value()) addVectorFileToQGIS(self.lneSavePolyFile.text(), atTop=True) self.cleanMessageBars(True) self.stackedWidget.setDisabled(False) QApplication.restoreOverrideCursor() self.iface.messageBar().popWidget() self.iface.mainWindow().statusBar().clearMessage() if result is not None: self.fraMain.setDisabled(False) self.send_to_messagebar(result, level=Qgis.Warning, duration=0, addToLog=False) return False # leave dialog open return super(PointTrailToPolygonDialog, self).accept(*args, **kwargs) except Exception as err: QApplication.restoreOverrideCursor() self.iface.mainWindow().statusBar().clearMessage() self.cleanMessageBars(True) self.stackedWidget.setDisabled(False) self.send_to_messagebar(str(err), level=Qgis.Critical, duration=0, addToLog=True, core_QGIS=False, showLogPanel=True, exc_info=sys.exc_info()) return False # leave dialog open