def test05_cleanTrimPoints(self):
global fileTrimmed, data_col
fileTrimmed = os.path.join(TmpDir, os.path.splitext(
os.path.basename(file_csv))[0] + '_normtrimmed.csv')
file_shp = os.path.join(TmpDir, os.path.splitext(
os.path.basename(file_csv))[0] + '_normtrimmed.shp')
file_removed = os.path.join(TmpDir, os.path.splitext(
os.path.basename(file_csv))[0] + '_remove.shp')
data_col = r'Yield'
gdf_points, gdfpts_crs = convert.convert_csv_to_points(file_csv,
coord_columns_epsg=4326,
out_epsg=epsg)
gdf_out, crs_out = clean_trim_points(gdf_points, gdfpts_crs, data_col, fileTrimmed,
out_keep_shapefile=file_shp,
out_removed_shapefile=file_removed,
boundary_polyfile=fileBox,
thin_dist_m=2.5)
self.assertTrue(os.path.exists(fileTrimmed))
self.assertTrue(os.path.exists(file_shp))
self.assertTrue(os.path.exists(file_removed))
self.assertEqual(gdf_out.crs, crs.from_epsg(epsg))
self.assertEqual(len(gdf_out), 648)
self.assertIn('nrm_' + data_col, gdf_out.columns)
self.assertIn('Easting', gdf_out.columns)
self.assertIn('Northing', gdf_out.columns)
self.assertIn('EN_EPSG', gdf_out.columns)
def test_convert_csv_to_points_WGS84(self):
file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")
out_epsg = 28354
out_file = os.path.join(
TmpDir,
os.path.basename(file).replace('.csv', '_{}.shp'.format(out_epsg)))
gdfData, gdfCRS = convert_csv_to_points(file,
out_file,
coord_columns_epsg=4326,
out_epsg=out_epsg)
self.assertTrue(os.path.exists(out_file))
self.assertIsInstance(gdfData, GeoDataFrame)
self.assertEqual(len(gdfData), 1543)
self.assertEqual(list(set(gdfData.geom_type)), ['Point'])
import numpy as np
np.testing.assert_almost_equal(
list(gdfData.total_bounds),
[598868.6709, 6054050.0529, 599242.9049, 6054415.3845], 4)
self.assertEqual(gdfCRS.epsg_number, out_epsg)
self.assertEqual(
gdfCRS.crs_wkt,
'PROJCS["GDA94 / MGA zone 54",GEOGCS["GDA94",DATUM["Geocentric_Datum_of_Australia_1994",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6283"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4283"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",141],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",10000000],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","28354"]]'
)
def test_cleanTrimPoints(self):
in_csv = os.path.join(this_dir + "/data/area2_yield_ISO-8859-1.csv")
in_poly = os.path.join(this_dir + "/data/area2_onebox_94mga54.shp")
out_csv = os.path.join(TmpDir, os.path.basename(in_csv))
out_shp = os.path.join(
TmpDir,
os.path.basename(in_csv).replace('.csv', '.shp'))
out_rm_shp = os.path.join(
TmpDir,
os.path.basename(in_csv).replace('.csv', '_remove.shp'))
gdf_points, gdf_pts_crs = convert.convert_csv_to_points(
in_csv, coord_columns_epsg=4326, out_epsg=28354)
out_gdf, out_crs = clean_trim_points(gdf_points,
gdf_pts_crs,
'Yld Mass(Dry)(tonne/ha)',
out_csv,
out_keep_shapefile=out_shp,
out_removed_shapefile=out_rm_shp,
boundary_polyfile=in_poly,
thin_dist_m=2.5)
self.assertIsInstance(out_gdf, GeoDataFrame)
self.assertTrue(os.path.exists(out_csv))
self.assertTrue(gdf_pts_crs, out_crs)
self.assertEqual(out_gdf.crs, {'init': 'epsg:28354', 'no_defs': True})
self.assertEqual(len(out_gdf), 554)
self.assertIn('EN_EPSG', out_gdf.columns)
def test_createPolygonFromPointTrail(self):
in_csv = os.path.join(this_dir + "/data/area2_yield_ISO-8859-1.csv")
# out_ptsfile = os.path.join(TmpDir, os.path.splitext(os.path.basename(in_csv))[0] + '_points.shp')
out_polyfile = os.path.join(
TmpDir,
os.path.splitext(os.path.basename(in_csv))[0] + '_poly.shp')
gdf_points, gdf_pts_crs = convert.convert_csv_to_points(
in_csv, None, coord_columns_epsg=4326, out_epsg=28354)
create_polygon_from_point_trail(gdf_points,
gdf_pts_crs,
out_polyfile,
thin_dist_m=2.5,
aggregate_dist_m=25,
buffer_dist_m=10,
shrink_dist_m=3)
self.assertTrue(os.path.exists(out_polyfile), True)
vect_desc = VectorDescribe(out_polyfile)
self.assertEqual(vect_desc.crs.epsg_number, 28354)
self.assertFalse(vect_desc.is_mz_aware)
self.assertEqual(vect_desc.geometry_type, 'Polygon')
def test1_prepareForVesperKrig_filesExist(self):
gdfPoints, gdfPtsCrs = convert.convert_csv_to_points(
file, coord_columns_epsg=4326, out_epsg=28354)
outGDF, outCRS = clean_trim_points(gdfPoints,
gdfPtsCrs,
data_col,
fileSubName + '_trimmed.csv',
poly,
thin_dist_m=2.5)
self.assertTrue(os.path.exists(fileSubName + '_trimmed.csv'))
self.assertEqual(outGDF.crs, {'init': 'epsg:28354', 'no_defs': True})
self.assertEqual(len(outGDF), 554)
processing.block_grid(in_shapefilename=poly,
pixel_size=5,
out_rasterfilename=block_tif,
out_vesperfilename=fileSubName + '_block_v.txt',
snap=True,
overwrite=True)
global file_ctrl
file_bat, file_ctrl = prepare_for_vesper_krige(
outGDF,
data_col,
fileSubName + '_block_v.txt',
TmpDir,
control_textfile=os.path.basename(fileSubName) + '_control.txt',
block_size=30,
coord_columns=[],
epsg=28354)
self.assertTrue(
os.path.exists(os.path.join(TmpDir, r'Vesper\Do_Vesper.bat')))
self.assertTrue(
os.path.exists(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) + '_control.txt')))
self.assertTrue(
os.path.exists(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) +
'_vesperdata.csv')))
dfCSV = pd.read_csv(
os.path.join(TmpDir, r'Vesper',
os.path.basename(fileSubName) + '_vesperdata.csv'))
x_column, y_column = predictCoordinateColumnNames(dfCSV.columns)
self.assertEqual(x_column.upper(), 'EASTING')
self.assertEqual(y_column.upper(), 'NORTHING')
def test_thinPointByDistance_mga54(self):
file = os.path.realpath(this_dir + "/data/area2_yield_ISO-8859-1.csv")
out_epsg = 28354
out_file = os.path.join(
TmpDir,
os.path.basename(file).replace('.csv', '_{}.shp'.format(out_epsg)))
gdf, gdfCRS = convert_csv_to_points(file,
out_file,
coord_columns_epsg=4326,
out_epsg=out_epsg)
result = thin_point_by_distance(gdf, gdfCRS, 2.5)
self.assertEqual(len(result), len(gdf))
self.assertEqual(len(result[result['filter'].isnull()]), 1434)
self.assertEqual(result.crs, gdf.crs)
def test_convert_csv_to_points_EastingNorthings(self):
file = os.path.realpath(this_dir + "/data/area1_yield_ascii_wgs84.csv")
epsg = 28354
out_file = os.path.join(
TmpDir,
os.path.basename(file).replace('.csv', '_{}.shp'.format(epsg)))
gdfData, gdfCRS = convert_csv_to_points(
file,
out_file,
coord_columns=['Easting', 'Northing'],
coord_columns_epsg=epsg)
self.assertIsInstance(gdfData, GeoDataFrame)
self.assertEqual(len(gdfData), 13756)
self.assertEqual(gdfCRS.epsg_number, epsg)
self.assertTrue(os.path.exists(out_file))
def test02_createPolygonFromPointTrail(self):
global filePoints, filePoly
filePoints = os.path.join(TmpDir,
os.path.splitext(os.path.basename(file_csv))[0] + '_points.shp')
filePoly = os.path.join(TmpDir,
os.path.splitext(os.path.basename(file_csv))[0] + '_poly.shp')
if not os.path.exists(filePoly):
gdf_points, gdf_pts_crs = convert.convert_csv_to_points(file_csv, filePoints,
coord_columns_epsg=4326,
out_epsg=epsg)
create_polygon_from_point_trail(gdf_points, gdf_pts_crs, filePoly,
thin_dist_m=2.5,
aggregate_dist_m=25,
buffer_dist_m=7,
shrink_dist_m=3)
self.assertTrue(os.path.exists(filePoly), True)
def accept(self, *args, **kwargs):
if not self.validate():
return False
try:
# disable form via a frame, this will still allow interaction with the message bar
self.stackedWidget.setDisabled(True)
# clean gui and Qgis messagebars
self.cleanMessageBars(True)
# self.iface.messageBar().clearWidgets()
# Change cursor to Wait cursor
QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
self.iface.mainWindow().statusBar().showMessage('Processing {}'.format(self.windowTitle()))
self.send_to_messagebar("Please wait.. QGIS will be locked... See log panel for progress.",
level=Qgis.Warning,
duration=0, addToLog=False, core_QGIS=False, showLogPanel=True)
gp_layer_name = ''
LOGGER.info('{st}\nProcessing {}'.format(self.windowTitle(), st='*' * 50))
# Add settings to log
settingsStr = 'Parameters:---------------------------------------'
if self.optFile.isChecked():
settingsStr += '\n {:30}\t{}'.format('File:', self.lneInCSVFile.text())
settingsStr += '\n {:30}\t{}, {}'.format('Geometry Fields:', self.cboXField.currentText(),
self.cboYField.currentText())
settingsStr += '\n {:30}\t{} - {}'.format('CSV Coordinate System:', self.qgsCRScsv.crs().authid(),
self.qgsCRScsv.crs().description())
else:
if self.chkUseSelected.isChecked():
settingsStr += '\n {:30}\t{} with {} selected features'.format('Layer:',
self.mcboTargetLayer.currentLayer().name(),
self.mcboTargetLayer.currentLayer().selectedFeatureCount())
else:
settingsStr += '\n {:30}\t{}'.format('Layer:', self.mcboTargetLayer.currentLayer().name())
crs_units = QgsUnitTypes.toString(self.mCRSoutput.crs().mapUnits())
settingsStr += '\n {:30}\t{} {}'.format('Thinning Distance:', self.dsbThinDist.value(),crs_units)
settingsStr += '\n {:30}\t{} {}'.format("Aggregate Distance:", self.dsbAggregateDist.value(),crs_units)
settingsStr += '\n {:30}\t{} {}'.format("Buffer Distance:", self.dsbBufferDist.value(),crs_units)
settingsStr += '\n {:30}\t{} {}'.format("Shrink Distance:", self.dsbShrinkDist.value(),crs_units)
settingsStr += '\n {:30}\t{}'.format('Output Polygon Shapefile:', self.lneSavePolyFile.text())
if self.lneSavePointsFile.text() == '':
settingsStr += '\n {:30}\t{}'.format('Saved Points Shapefile:', self.lneSavePointsFile.text())
settingsStr += '\n {:30}\t{} - {}\n\n'.format('Output Projected Coordinate System:',
self.mCRSoutput.crs().authid(),
self.mCRSoutput.crs().description())
LOGGER.info(settingsStr)
stepTime = time.time()
if self.optFile.isChecked():
in_epsg = int(self.qgsCRScsv.crs().authid().replace('EPSG:',''))
in_crs = self.qgsCRScsv.crs()
else:
in_epsg =self.mcboTargetLayer.currentLayer().crs().authid().replace('EPSG:','')
in_crs = self.mcboTargetLayer.currentLayer().crs()
out_epsg = int(self.mCRSoutput.crs().authid().replace('EPSG:',''))
filePoly = None
gdfPoints = None
filePoints = None
if self.optFile.isChecked():
if self.DEBUG:
filePoints = os.path.join(TEMPDIR, os.path.splitext(os.path.basename(self.lneSavePolyFile.text()))[0] + '_table2pts.shp')
if os.path.splitext(self.lneInCSVFile.text())[-1] == '.csv':
gdfPoints, gdfPtsCrs = convert.convert_csv_to_points(self.lneInCSVFile.text() , out_shapefilename=filePoints,
coord_columns=[self.cboXField.currentText(),
self.cboYField.currentText()],
coord_columns_epsg=in_epsg)
elif os.path.splitext(self.lneInCSVFile.text())[-1] in ['.xls', '.xlsx', '.ods']:
xls_file = pd.ExcelFile(self.lneInCSVFile.text())
pdfxls = xls_file.parse(self.sheet(), skiprows=self.linesToIgnore() - 1)
del xls_file
gdfPoints, gdfPtsCrs = convert.add_point_geometry_to_dataframe(pdfxls,
coord_columns=[
self.cboXField.currentText(),
self.cboYField.currentText()],
coord_columns_epsg=in_epsg)
del pdfxls
LOGGER.info('{:<30} {d:<15} {}'.format('Add Geometry to Table','',
d=str(timedelta(seconds=time.time() - stepTime))))
stepTime = time.time()
if filePoints is not None:
describe.save_geopandas_tofile(gdfPoints, filePoints) #, file_encoding=self.file_encoding)
if self.DISP_TEMP_LAYERS and filePoints is not None:
addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0],
group_layer_name='DEBUG', atTop=True)
else:
layerPts = self.mcboTargetLayer.currentLayer()
if layerPts.providerType() == 'delimitedtext' or \
os.path.splitext(get_layer_source(layerPts))[-1] == '.vrt' or \
self.chkUseSelected.isChecked() or self.optFile.isChecked():
filePoints = os.path.join(TEMPDIR, "{}_points.shp".format(layerPts.name()))
if self.chkUseSelected.isChecked():
filePoints = os.path.join(TEMPDIR, "{}_selected_points.shp".format(layerPts.name()))
if os.path.exists(filePoints):
removeFileFromQGIS(filePoints)
ptsLayer = copyLayerToMemory(layerPts, layerPts.name() + "_memory", bAddUFI=True,
bOnlySelectedFeat=self.chkUseSelected.isChecked())
_writer = QgsVectorFileWriter.writeAsVectorFormat(ptsLayer, filePoints, "utf-8",
self.mCRSoutput.crs(), driverName="ESRI Shapefile")
LOGGER.info('{:<30} {d:<15} {}'.format('Save layer/selection to file',filePoints,
d=str(timedelta(seconds=time.time() - stepTime) )))
stepTime = time.time()
del ptsLayer
if self.DISP_TEMP_LAYERS:
addVectorFileToQGIS(filePoints, layer_name=os.path.splitext(os.path.basename(filePoints))[0],
group_layer_name='DEBUG', atTop=True)
else:
filePoints = get_layer_source(layerPts)
if gdfPoints is None:
ptsDesc = describe.VectorDescribe(filePoints)
gdfPtsCrs = ptsDesc.crs
gdfPoints = ptsDesc.open_geo_dataframe()
if in_crs.authid() != self.mCRSoutput.crs().authid():
gdfPoints = gdfPoints.to_crs(epsg=out_epsg)
gdfPtsCrs = pyprecag_crs.crs()
gdfPtsCrs.getFromEPSG(out_epsg)
LOGGER.info('{:<30} {d:<15} {} to {}'.format('Reproject points', in_crs.authid(),
self.mCRSoutput.crs().authid(),
d=str(timedelta(seconds=time.time() - stepTime))))
if self.DEBUG:
filePoints = os.path.join(TEMPDIR, os.path.basename(self.lneSavePolyFile.text().replace('.csv', '_ptsprj.shp')))
removeFileFromQGIS(filePoints)
describe.save_geopandas_tofile(gdfPoints, filePoints)
if self.DISP_TEMP_LAYERS:
if self.DEBUG:
addVectorFileToQGIS(filePoints,
layer_name=os.path.splitext(os.path.basename(filePoints))[0],
group_layer_name='DEBUG', atTop=True)
else:
addVectorFileToQGIS(filePoints,
layer_name=os.path.splitext(os.path.basename(filePoints))[0],
atTop=True)
stepTime = time.time()
result = processing.create_polygon_from_point_trail(gdfPoints, gdfPtsCrs,
out_filename=self.lneSavePolyFile.text(),
thin_dist_m=self.dsbThinDist.value(),
aggregate_dist_m=self.dsbAggregateDist.value(),
buffer_dist_m=self.dsbBufferDist.value(),
shrink_dist_m=self.dsbShrinkDist.value())
addVectorFileToQGIS(self.lneSavePolyFile.text(), atTop=True)
self.cleanMessageBars(True)
self.stackedWidget.setDisabled(False)
QApplication.restoreOverrideCursor()
self.iface.messageBar().popWidget()
self.iface.mainWindow().statusBar().clearMessage()
if result is not None:
self.fraMain.setDisabled(False)
self.send_to_messagebar(result, level=Qgis.Warning, duration=0, addToLog=False)
return False # leave dialog open
return super(PointTrailToPolygonDialog, self).accept(*args, **kwargs)
except Exception as err:
QApplication.restoreOverrideCursor()
self.iface.mainWindow().statusBar().clearMessage()
self.cleanMessageBars(True)
self.stackedWidget.setDisabled(False)
self.send_to_messagebar(str(err), level=Qgis.Critical,
duration=0, addToLog=True, core_QGIS=False, showLogPanel=True,
exc_info=sys.exc_info())
return False # leave dialog open