def test_delete_point_cloud_feature(celery_task_always_eager, projects_fixture,
point_cloud_fixture,
lidar_las1pt2_file_fixture):
PointCloudService.fromFileObj(point_cloud_fixture.id,
lidar_las1pt2_file_fixture,
lidar_las1pt2_file_fixture.name)
point_cloud = db_session.query(PointCloud).get(1)
feature_asset_path = get_asset_path(point_cloud.feature.assets[0].path)
FeaturesService.delete(point_cloud.feature.id)
assert db_session.query(PointCloud).count() == 1
assert db_session.query(PointCloud).get(1).feature is None
assert db_session.query(Feature).count() == 0
assert db_session.query(FeatureAsset).count() == 0
assert os.path.exists(
get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR))
assert not os.path.exists(feature_asset_path)
def test_add_point_cloud_file(projects_fixture, point_cloud_fixture,
lidar_las1pt2_file_fixture,
convert_to_potree_mock, check_point_cloud_mock,
get_point_cloud_info_mock):
filename = os.path.basename(lidar_las1pt2_file_fixture.name)
task = PointCloudService.fromFileObj(point_cloud_fixture.id,
lidar_las1pt2_file_fixture, filename)
assert task.status == "RUNNING"
assert point_cloud_fixture.task_id == task.id
# load updated point cloud
point_cloud = db_session.query(PointCloud).get(1)
las_files = os.listdir(
get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR))
assert len(las_files) == 1
assert las_files[0] == os.path.basename(filename)
original_file_size = os.fstat(lidar_las1pt2_file_fixture.fileno()).st_size
asset_file_path = os.path.join(
get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR),
filename)
assert os.path.getsize(asset_file_path) == original_file_size
# run conversion tool (that we had mocked)
_, convert_kwargs = convert_to_potree_mock.apply_async.call_args
assert projects_fixture.id == convert_kwargs['args'][0]
convert_to_potree(projects_fixture.id)
# load updated point cloud
point_cloud = db_session.query(PointCloud).get(1)
assert point_cloud.task.status == "FINISHED"
assert db_session.query(Feature).count() == 1
assert db_session.query(FeatureAsset).count() == 1
assert len(os.listdir(get_project_asset_dir(point_cloud.project_id))) == 2
assert len(os.listdir(get_asset_path(point_cloud.feature.assets[0].path))
) == 5 # index.html, preview.html, pointclouds, libs, logo
assert os.path.isfile(
os.path.join(get_asset_path(point_cloud.feature.assets[0].path),
"preview.html"))
with open(
os.path.join(get_asset_path(point_cloud.feature.assets[0].path),
"preview.html"), 'r+') as f:
preview = f.read()
assert "nsf_logo" not in preview
assert "$('.potree_menu_toggle').hide()" in preview
def test_create_feature_image_small_image(projects_fixture, image_small_DES_2176_fixture):
feature = FeaturesService.fromImage(projects_fixture.id, image_small_DES_2176_fixture, metadata={})
assert feature.project_id == projects_fixture.id
assert len(feature.assets) == 1
assert db_session.query(Feature).count() == 1
assert db_session.query(FeatureAsset).count() == 1
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 2
os.path.isfile(get_asset_path(feature.assets[0].path))
os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".thumb.jpeg"))
def test_create_feature_video_asset(projects_fixture, feature_fixture, video_file_fixture):
feature = FeaturesService.createFeatureAsset(projects_fixture.id,
feature_fixture.id,
FileStorage(video_file_fixture))
assert feature.id == feature_fixture.id
assert len(feature.assets) == 1
assert db_session.query(FeatureAsset).count() == 1
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1
os.path.isfile(get_asset_path(feature.assets[0].path))
os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".mp4"))
def test_import_point_clouds_from_agave(MockAgaveUtils, projects_fixture,
point_cloud_fixture,
lidar_las1pt2_file_fixture):
MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture
u1 = db_session.query(User).get(1)
files = [{"system": "designsafe.storage.default", "path": "file1.las"}]
import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id)
point_cloud = point_cloud_fixture
assert point_cloud.task.status == "FINISHED"
assert point_cloud.task.description == ""
assert len(os.listdir(get_project_asset_dir(point_cloud.project_id))) == 2
assert len(os.listdir(get_asset_path(point_cloud.feature.assets[0].path))
) == 5 # index.html, preview.html, pointclouds, libs, logo
assert len(
os.listdir(
get_asset_path(point_cloud_fixture.path,
PointCloudService.ORIGINAL_FILES_DIR))) == 1
def get_point_cloud_files(path):
"""
Get all point cloud files in a path
:param path: strings
:return: list of file paths of point cloud files
"""
from geoapi.services.point_cloud import PointCloudService
input_files = [
get_asset_path(path, file) for file in os.listdir(path)
if pathlib.Path(file).suffix.lstrip('.').lower() in
PointCloudService.LIDAR_FILE_EXTENSIONS
]
return input_files
def get_point_cloud_info(pointCloudId: int) -> dict:
"""
Get info on las files
:param pointCloudId: int
:return: None
"""
from geoapi.services.point_cloud import PointCloudService
point_cloud = PointCloudService.get(pointCloudId)
path_to_original_point_clouds = get_asset_path(
point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR)
input_files = get_point_cloud_files(path_to_original_point_clouds)
return [{'name': os.path.basename(f)} for f in input_files]
def putPointCloudInOriginalsFileDir(point_cloud_path: str, fileObj: IO, fileName: str):
""" Put file object in original files directory
:param point_cloud_path: str
:param fileObj: IO
:param fileName: str
:return: path to point cloud
"""
file_path = get_asset_path(point_cloud_path,
PointCloudService.ORIGINAL_FILES_DIR,
os.path.basename(fileName))
with open(file_path, "wb") as f:
# set current file position to start so all contents are copied.
fileObj.seek(0)
shutil.copyfileobj(fileObj, f)
return file_path
def test_import_point_clouds_from_agave_conversion_error(
MockAgaveUtils, convert_mock, projects_fixture, point_cloud_fixture,
lidar_las1pt2_file_fixture):
MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture
convert_mock.apply.side_effect = Exception("dummy")
u1 = db_session.query(User).get(1)
files = [{"system": "designsafe.storage.default", "path": "file1.las"}]
import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id)
point_cloud = point_cloud_fixture
assert point_cloud.task.status == "FAILED"
assert point_cloud.task.description == ""
assert len(
os.listdir(
get_asset_path(point_cloud_fixture.path,
PointCloudService.ORIGINAL_FILES_DIR))) == 1
def test_import_point_clouds_from_agave_check_point_cloud_missing_crs(
MockAgaveUtils, check_mock, projects_fixture, point_cloud_fixture,
lidar_las1pt2_file_fixture):
MockAgaveUtils().getFile.return_value = lidar_las1pt2_file_fixture
check_mock.apply.side_effect = InvalidCoordinateReferenceSystem()
u1 = db_session.query(User).get(1)
files = [{"system": "designsafe.storage.default", "path": "file1.las"}]
import_point_clouds_from_agave(u1.id, files, point_cloud_fixture.id)
point_cloud = point_cloud_fixture
assert point_cloud.task.status == "FAILED"
assert point_cloud.task.description == "Error importing file1.las: missing coordinate reference system"
assert len(
os.listdir(
get_asset_path(point_cloud_fixture.path,
PointCloudService.ORIGINAL_FILES_DIR))) == 0
def convert_to_potree(self, pointCloudId: int) -> None:
"""
Use the potree converter to convert a LAS/LAZ file to potree format
:param pointCloudId: int
:return: None
"""
from geoapi.models import Feature, FeatureAsset
from geoapi.services.point_cloud import PointCloudService
point_cloud = PointCloudService.get(pointCloudId)
path_to_original_point_clouds = get_asset_path(
point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR)
path_temp_processed_point_cloud_path = get_asset_path(
point_cloud.path, PointCloudService.PROCESSED_DIR)
input_files = [
get_asset_path(path_to_original_point_clouds, file)
for file in os.listdir(path_to_original_point_clouds)
if pathlib.Path(file).suffix.lstrip('.').lower() in
PointCloudService.LIDAR_FILE_EXTENSIONS
]
outline = get_bounding_box_2d(input_files)
command = [
"PotreeConverter", "--verbose", "-i", path_to_original_point_clouds,
"-o", path_temp_processed_point_cloud_path, "--overwrite",
"--generate-page", "index"
]
if point_cloud.conversion_parameters:
command.extend(point_cloud.conversion_parameters.split())
logger.info("Processing point cloud (#{}): {}".format(
pointCloudId, " ".join(command)))
subprocess.run(command, check=True, capture_output=True, text=True)
# Create preview viewer html (with no menu and now nsf logo)
with open(
os.path.join(path_temp_processed_point_cloud_path, "preview.html"),
'w+') as preview:
with open(
os.path.join(path_temp_processed_point_cloud_path,
"index.html"), 'r') as viewer:
content = viewer.read()
content = re.sub(r"<div class=\"nsf_logo\"(.+?)</div>",
'',
content,
flags=re.DOTALL)
content = content.replace("viewer.toggleSidebar()",
"$('.potree_menu_toggle').hide()")
preview.write(content)
if point_cloud.feature_id:
feature = point_cloud.feature
else:
feature = Feature()
feature.project_id = point_cloud.project_id
asset_uuid = uuid.uuid4()
base_filepath = make_project_asset_dir(point_cloud.project_id)
asset_path = os.path.join(base_filepath, str(asset_uuid))
fa = FeatureAsset(uuid=asset_uuid,
asset_type="point_cloud",
path=get_asset_relative_path(asset_path),
feature=feature)
feature.assets.append(fa)
point_cloud.feature = feature
feature.the_geom = from_shape(geometries.convert_3D_2D(outline), srid=4326)
point_cloud.task.status = "FINISHED"
point_cloud.task.description = ""
point_cloud_asset_path = get_asset_path(feature.assets[0].path)
shutil.rmtree(point_cloud_asset_path, ignore_errors=True)
shutil.move(path_temp_processed_point_cloud_path, point_cloud_asset_path)
try:
db_session.add(point_cloud)
db_session.add(feature)
db_session.commit()
except:
db_session.rollback()
raise
