def test_delete_point_cloud_feature(celery_task_always_eager, projects_fixture, point_cloud_fixture, lidar_las1pt2_file_fixture): PointCloudService.fromFileObj(point_cloud_fixture.id, lidar_las1pt2_file_fixture, lidar_las1pt2_file_fixture.name) point_cloud = db_session.query(PointCloud).get(1) feature_asset_path = get_asset_path(point_cloud.feature.assets[0].path) FeaturesService.delete(point_cloud.feature.id) assert db_session.query(PointCloud).count() == 1 assert db_session.query(PointCloud).get(1).feature is None assert db_session.query(Feature).count() == 0 assert db_session.query(FeatureAsset).count() == 0 assert os.path.exists( get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR)) assert not os.path.exists(feature_asset_path)
def test_add_point_cloud_file(projects_fixture, point_cloud_fixture, lidar_las1pt2_file_fixture, convert_to_potree_mock, check_point_cloud_mock, get_point_cloud_info_mock): filename = os.path.basename(lidar_las1pt2_file_fixture.name) task = PointCloudService.fromFileObj(point_cloud_fixture.id, lidar_las1pt2_file_fixture, filename) assert task.status == "RUNNING" assert point_cloud_fixture.task_id == task.id # load updated point cloud point_cloud = db_session.query(PointCloud).get(1) las_files = os.listdir( get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR)) assert len(las_files) == 1 assert las_files[0] == os.path.basename(filename) original_file_size = os.fstat(lidar_las1pt2_file_fixture.fileno()).st_size asset_file_path = os.path.join( get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR), filename) assert os.path.getsize(asset_file_path) == original_file_size # run conversion tool (that we had mocked) _, convert_kwargs = convert_to_potree_mock.apply_async.call_args assert projects_fixture.id == convert_kwargs['args'][0] convert_to_potree(projects_fixture.id) # load updated point cloud point_cloud = db_session.query(PointCloud).get(1) assert point_cloud.task.status == "FINISHED" assert db_session.query(Feature).count() == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(point_cloud.project_id))) == 2 assert len(os.listdir(get_asset_path(point_cloud.feature.assets[0].path)) ) == 5 # index.html, preview.html, pointclouds, libs, logo assert os.path.isfile( os.path.join(get_asset_path(point_cloud.feature.assets[0].path), "preview.html")) with open( os.path.join(get_asset_path(point_cloud.feature.assets[0].path), "preview.html"), 'r+') as f: preview = f.read() assert "nsf_logo" not in preview assert "$('.potree_menu_toggle').hide()" in preview
def test_create_feature_image_small_image(projects_fixture, image_small_DES_2176_fixture): feature = FeaturesService.fromImage(projects_fixture.id, image_small_DES_2176_fixture, metadata={}) assert feature.project_id == projects_fixture.id assert len(feature.assets) == 1 assert db_session.query(Feature).count() == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 2 os.path.isfile(get_asset_path(feature.assets[0].path)) os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".thumb.jpeg"))
def test_create_feature_video_asset(projects_fixture, feature_fixture, video_file_fixture): feature = FeaturesService.createFeatureAsset(projects_fixture.id, feature_fixture.id, FileStorage(video_file_fixture)) assert feature.id == feature_fixture.id assert len(feature.assets) == 1 assert db_session.query(FeatureAsset).count() == 1 assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1 os.path.isfile(get_asset_path(feature.assets[0].path)) os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".mp4"))
def test_import_point_clouds_from_agave(MockAgaveUtils, projects_fixture, point_cloud_fixture, lidar_las1pt2_file_fixture): MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture u1 = db_session.query(User).get(1) files = [{"system": "designsafe.storage.default", "path": "file1.las"}] import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id) point_cloud = point_cloud_fixture assert point_cloud.task.status == "FINISHED" assert point_cloud.task.description == "" assert len(os.listdir(get_project_asset_dir(point_cloud.project_id))) == 2 assert len(os.listdir(get_asset_path(point_cloud.feature.assets[0].path)) ) == 5 # index.html, preview.html, pointclouds, libs, logo assert len( os.listdir( get_asset_path(point_cloud_fixture.path, PointCloudService.ORIGINAL_FILES_DIR))) == 1
def get_point_cloud_files(path): """ Get all point cloud files in a path :param path: strings :return: list of file paths of point cloud files """ from geoapi.services.point_cloud import PointCloudService input_files = [ get_asset_path(path, file) for file in os.listdir(path) if pathlib.Path(file).suffix.lstrip('.').lower() in PointCloudService.LIDAR_FILE_EXTENSIONS ] return input_files
def get_point_cloud_info(pointCloudId: int) -> dict: """ Get info on las files :param pointCloudId: int :return: None """ from geoapi.services.point_cloud import PointCloudService point_cloud = PointCloudService.get(pointCloudId) path_to_original_point_clouds = get_asset_path( point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR) input_files = get_point_cloud_files(path_to_original_point_clouds) return [{'name': os.path.basename(f)} for f in input_files]
def putPointCloudInOriginalsFileDir(point_cloud_path: str, fileObj: IO, fileName: str): """ Put file object in original files directory :param point_cloud_path: str :param fileObj: IO :param fileName: str :return: path to point cloud """ file_path = get_asset_path(point_cloud_path, PointCloudService.ORIGINAL_FILES_DIR, os.path.basename(fileName)) with open(file_path, "wb") as f: # set current file position to start so all contents are copied. fileObj.seek(0) shutil.copyfileobj(fileObj, f) return file_path
def test_import_point_clouds_from_agave_conversion_error( MockAgaveUtils, convert_mock, projects_fixture, point_cloud_fixture, lidar_las1pt2_file_fixture): MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture convert_mock.apply.side_effect = Exception("dummy") u1 = db_session.query(User).get(1) files = [{"system": "designsafe.storage.default", "path": "file1.las"}] import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id) point_cloud = point_cloud_fixture assert point_cloud.task.status == "FAILED" assert point_cloud.task.description == "" assert len( os.listdir( get_asset_path(point_cloud_fixture.path, PointCloudService.ORIGINAL_FILES_DIR))) == 1
def test_import_point_clouds_from_agave_check_point_cloud_missing_crs( MockAgaveUtils, check_mock, projects_fixture, point_cloud_fixture, lidar_las1pt2_file_fixture): MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture check_mock.apply.side_effect = InvalidCoordinateReferenceSystem() u1 = db_session.query(User).get(1) files = [{"system": "designsafe.storage.default", "path": "file1.las"}] import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id) point_cloud = point_cloud_fixture assert point_cloud.task.status == "FAILED" assert point_cloud.task.description == "Error importing file1.las: missing coordinate reference system" assert len( os.listdir( get_asset_path(point_cloud_fixture.path, PointCloudService.ORIGINAL_FILES_DIR))) == 0
def convert_to_potree(self, pointCloudId: int) -> None: """ Use the potree converter to convert a LAS/LAZ file to potree format :param pointCloudId: int :return: None """ from geoapi.models import Feature, FeatureAsset from geoapi.services.point_cloud import PointCloudService point_cloud = PointCloudService.get(pointCloudId) path_to_original_point_clouds = get_asset_path( point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR) path_temp_processed_point_cloud_path = get_asset_path( point_cloud.path, PointCloudService.PROCESSED_DIR) input_files = [ get_asset_path(path_to_original_point_clouds, file) for file in os.listdir(path_to_original_point_clouds) if pathlib.Path(file).suffix.lstrip('.').lower() in PointCloudService.LIDAR_FILE_EXTENSIONS ] outline = get_bounding_box_2d(input_files) command = [ "PotreeConverter", "--verbose", "-i", path_to_original_point_clouds, "-o", path_temp_processed_point_cloud_path, "--overwrite", "--generate-page", "index" ] if point_cloud.conversion_parameters: command.extend(point_cloud.conversion_parameters.split()) logger.info("Processing point cloud (#{}): {}".format( pointCloudId, " ".join(command))) subprocess.run(command, check=True, capture_output=True, text=True) # Create preview viewer html (with no menu and now nsf logo) with open( os.path.join(path_temp_processed_point_cloud_path, "preview.html"), 'w+') as preview: with open( os.path.join(path_temp_processed_point_cloud_path, "index.html"), 'r') as viewer: content = viewer.read() content = re.sub(r"<div class=\"nsf_logo\"(.+?)</div>", '', content, flags=re.DOTALL) content = content.replace("viewer.toggleSidebar()", "$('.potree_menu_toggle').hide()") preview.write(content) if point_cloud.feature_id: feature = point_cloud.feature else: feature = Feature() feature.project_id = point_cloud.project_id asset_uuid = uuid.uuid4() base_filepath = make_project_asset_dir(point_cloud.project_id) asset_path = os.path.join(base_filepath, str(asset_uuid)) fa = FeatureAsset(uuid=asset_uuid, asset_type="point_cloud", path=get_asset_relative_path(asset_path), feature=feature) feature.assets.append(fa) point_cloud.feature = feature feature.the_geom = from_shape(geometries.convert_3D_2D(outline), srid=4326) point_cloud.task.status = "FINISHED" point_cloud.task.description = "" point_cloud_asset_path = get_asset_path(feature.assets[0].path) shutil.rmtree(point_cloud_asset_path, ignore_errors=True) shutil.move(path_temp_processed_point_cloud_path, point_cloud_asset_path) try: db_session.add(point_cloud) db_session.add(feature) db_session.commit() except: db_session.rollback() raise