def addGeoJSON(projectId: int, feature: Dict, original_path=None) -> List[Feature]: """ Add a GeoJSON feature to a project :param projectId: int :param feature: dict :return: Feature """ try: data = geojson.loads(json.dumps(feature)) except ValueError: raise InvalidGeoJSON features = [] if data["type"] == "Feature": feat = Feature.fromGeoJSON(data) feat.project_id = projectId # strip out image_src, thumb_src if they are there from the old hazmapper geojson feat = FeaturesService._importHazmapperV1Images(feat) db_session.add(feat) features.append(feat) elif data["type"] == "FeatureCollection": fc = geojson.FeatureCollection(data) for feature in fc.features: feat = Feature.fromGeoJSON(feature) feat.project_id = projectId feat = FeaturesService._importHazmapperV1Images(feat) db_session.add(feat) features.append(feat) else: raise InvalidGeoJSON( "Valid GeoJSON must be either a Feature or FeatureCollection.") db_session.commit() return features
def fromFileObj(pointCloudId: int, fileObj: IO, fileName: str): """ Add a point cloud file When point cloud file has been processed, a feature will be created/updated with feature asset containing processed point cloud Different processing steps are applied asynchronously by default. :param pointCloudId: int :param fileObj: IO :param fileName: str :return: processingTask: Task """ PointCloudService.check_file_extension(fileName) point_cloud = PointCloudService.get(pointCloudId) file_path = PointCloudService.putPointCloudInOriginalsFileDir(point_cloud.path, fileObj, fileName); try: result = check_point_cloud.apply_async(args=[file_path]) result.get(); except InvalidCoordinateReferenceSystem as e: os.remove(file_path) logger.error("Point cloud file ({}) missing required coordinate reference system".format(file_path)) raise e result = get_point_cloud_info.apply_async(args=[pointCloudId]) point_cloud.files_info = json.dumps(result.get()) db_session.add(point_cloud) db_session.commit() return PointCloudService._process_point_clouds(pointCloudId)
def fromShapefile(projectId: int, fileObj: IO, metadata: Dict, additional_files: List[IO], original_path=None) -> Feature: """ Create features from shapefile :param projectId: int :param fileObj: file descriptor :param additional_files: file descriptor for all the other non-.shp files :param metadata: Dict of <key, val> pairs [IGNORED} :param original_path: str path of original file location [IGNORED} :return: Feature """ features = [] for geom, properties in VectorService.process_shapefile( fileObj, additional_files): feat = Feature() feat.project_id = projectId feat.the_geom = from_shape(geometries.convert_3D_2D(geom), srid=4326) feat.properties = properties db_session.add(feat) features.append(feat) db_session.commit() return features
def addOverlay(projectId: int, fileObj: IO, bounds: List[float], label: str) -> Overlay: """ :param projectId: int :param fileObj: IO :param bounds: List [minLon, minLat, maxLon, maxLat] :param label: str :return: None """ imdata = ImageService.processOverlay(fileObj) ov = Overlay() ov.label = label ov.minLon = bounds[0] ov.minLat = bounds[1] ov.maxLon = bounds[2] ov.maxLat = bounds[3] ov.project_id = projectId ov.uuid = uuid.uuid4() asset_path = os.path.join(str(make_project_asset_dir(projectId)), str(ov.uuid) + '.jpeg') ov.path = get_asset_relative_path(asset_path) imdata.original.save(asset_path, 'JPEG') imdata.thumb.save( pathlib.Path(asset_path).with_suffix(".thumb.jpeg"), "JPEG") db_session.add(ov) db_session.commit() return ov
def _process_point_clouds(pointCloudId: int) -> Task: """ Process point cloud files :param pointCloudId: int :return: processingTask: Task """ point_cloud = PointCloudService.get(pointCloudId) celery_task_id = celery_uuid() task = Task() task.process_id = celery_task_id task.status = "RUNNING" task.description = "Processing point cloud #{}".format(pointCloudId) point_cloud.task = task db_session.add(task) db_session.add(point_cloud) db_session.commit() logger.info("Starting potree processing task (#{}: '{}') for point cloud (#{}).".format( task.id, celery_task_id, pointCloudId)) # Process asynchronously lidar file and add a feature asset convert_to_potree.apply_async(args=[pointCloudId], task_id=celery_task_id) return task
def feature_fixture(): home = os.path.dirname(__file__) with open(os.path.join(home, 'fixtures/properties.json'), 'rb') as f: feat = Feature.fromGeoJSON(json.loads(f.read())) feat.project_id = 1 db_session.add(feat) db_session.commit() yield feat
def on_failure(self, exc, task_id, args, kwargs, einfo): logger.info("Task ({}, point cloud {}) failed: {}".format( task_id, args, exc)) failed_task = db_session.query(Task).filter( Task.process_id == task_id).first() failed_task.status = "FAILED" failed_task.description = "" db_session.add(failed_task) db_session.commit()
def fromLatLng(projectId: int, lat: float, lng: float, metadata: Dict) -> Feature: point = Point(lng, lat) f = Feature() f.project_id = projectId f.the_geom = from_shape(point, srid=4326) f.properties = metadata or {} db_session.add(f) db_session.commit() return f
def projects_fixture2(): proj = Project(name="test2", description="description2") u1 = db_session.query(User).filter(User.username == "test1").first() proj.users.append(u1) proj.tenant_id = u1.tenant_id db_session.add(proj) db_session.commit() yield proj shutil.rmtree(get_project_asset_dir(proj.id), ignore_errors=True)
def create(username: str, tenant: str, jwt: str = None) -> User: """ :rtype: User """ u = User(username=username, tenant_id=tenant) if jwt: u.jwt = jwt db_session.add(u) db_session.commit() return u
def create(user: User, status: AnyStr, message: AnyStr) -> Notification: note = Notification(username=user.username, tenant_id=user.tenant_id, status=status, message=message) try: db_session.add(note) db_session.commit() return note except Exception: db_session.rollback() raise
def addRandomMarkers(): proj = Project(name="test", description="test", tenant_id="designsafe") user = db_session.query(User).filter(User.username == "jmeiring").first() proj.users.append(user) db_session.add(user) for i in range(0, 10000): p = Point(random.uniform(-180, 180), random.uniform(-90, 90)) feat = Feature( the_geom=from_shape(p, srid=4326), ) feat.project = proj db_session.add(feat) db_session.commit()
def _update_point_cloud_task(pointCloudId: int, description: str = None, status: str = None): task = pointcloud.PointCloudService.get(pointCloudId).task if description is not None: task.description = description if status is not None: task.status = status try: db_session.add(task) db_session.commit() except Exception: db_session.rollback() raise
def fromImage(projectId: int, fileObj: IO, metadata: Dict, original_path: str = None) -> Feature: """ Create a Point feature from a georeferenced image :param projectId: int :param fileObj: file :param metadata: dict :return: None """ imdata = ImageService.processImage(fileObj) point = Point(imdata.coordinates) f = Feature() f.project_id = projectId f.the_geom = from_shape(point, srid=4326) f.properties = metadata asset_uuid = uuid.uuid4() base_filepath = make_project_asset_dir(projectId) asset_path = os.path.join(base_filepath, str(asset_uuid) + '.jpeg') fa = FeatureAsset( uuid=asset_uuid, asset_type="image", original_path=original_path, display_path=original_path, path=get_asset_relative_path(asset_path), feature=f, ) f.assets.append(fa) thumbnail_path = os.path.join(base_filepath, str(asset_uuid) + ".thumb.jpeg") resized_image_path = os.path.join(base_filepath, str(asset_uuid) + '.jpeg') try: imdata.thumb.save(thumbnail_path, "JPEG") imdata.resized.save(resized_image_path, "JPEG") except: if os.path.exists(thumbnail_path): os.remove(thumbnail_path) if os.path.exists(resized_image_path): os.remove(resized_image_path) raise finally: fileObj.close() db_session.add(f) db_session.commit() return f
def makeObservable(proj: Project, user: User, watch_content: bool): """ Makes a project an observable project Requires project's system_path, system_id, tenant_id to exist :param proj: Project :param user: User :param watch_content: bool :return: None """ folder_name = Path(proj.system_path).name name = proj.system_id + '/' + folder_name # TODO: Handle no storage system found system = AgaveUtils(user.jwt).systemsGet(proj.system_id) obs = ObservableDataProject(system_id=proj.system_id, path=proj.system_path, watch_content=watch_content) users = get_system_users(proj.tenant_id, user.jwt, proj.system_id) logger.info( "Updating project:{} to have the following users: {}".format( name, users)) project_users = [ UserService.getOrCreateUser(u, tenant=proj.tenant_id) for u in users ] proj.users = project_users obs.project = proj try: db_session.add(obs) db_session.commit() except IntegrityError as e: db_session.rollback() logger.exception( "User:{} tried to create an observable project that already exists: '{}'" .format(user.username, name)) raise ObservableProjectAlreadyExists( "'{}' project already exists".format(name)) if watch_content: import_from_agave.apply_async(args=[ obs.project.tenant_id, user.id, obs.system_id, obs.path, obs.project_id ])
def addTileServer(projectId: int, data: Dict): """ :param projectId: int :param data: Dict :return: ts: TileServer """ ts = TileServer() for key, value in data.items(): setattr(ts, key, value) ts.project_id = projectId db_session.add(ts) db_session.commit() return ts
def observable_projects_fixture(): u1 = db_session.query(User).filter(User.username == "test1").first() proj = Project(name="test_observable", description="description", tenant_id=u1.tenant_id) obs = ObservableDataProject( system_id="testSystem", path="/testPath", watch_content=True ) obs.project = proj proj.users.append(u1) db_session.add(obs) db_session.add(proj) db_session.commit() yield obs shutil.rmtree(get_project_asset_dir(proj.id), ignore_errors=True)
def fromGPX(projectId: int, fileObj: IO, metadata: Dict, original_path=None) -> Feature: # TODO: Fiona should support reading from the file directly, this MemoryFile business # should not be needed with fiona.io.MemoryFile(fileObj) as memfile: with memfile.open(layer="tracks") as collection: track = collection[0] shp = shape(track["geometry"]) feat = Feature() feat.project_id = projectId feat.the_geom = from_shape(geometries.convert_3D_2D(shp), srid=4326) feat.properties = metadata or {} db_session.add(feat) db_session.commit() return feat
def create(projectId: int, data: dict, user: User) -> PointCloud: """ Create a PointCloud for a user. :param projectId: int :param data: dict :param user: User :return: PointCloud """ point_cloud_uid = uuid.uuid4() point_cloud_path = os.path.join(make_project_asset_dir(projectId), str(point_cloud_uid)) file_point_cloud_path = os.path.join(point_cloud_path, PointCloudService.ORIGINAL_FILES_DIR) pathlib.Path(file_point_cloud_path).mkdir(parents=True, exist_ok=True) point_cloud = PointCloud(**data) point_cloud.project_id = projectId point_cloud.tenant_id = user.tenant_id point_cloud.uuid = point_cloud_uid point_cloud.path = get_asset_relative_path(point_cloud_path) db_session.add(point_cloud) db_session.commit() return point_cloud
def refresh_observable_projects(): try: obs = db_session.query(ObservableDataProject).all() for i, o in enumerate(obs): # we need a user with a jwt for importing importing_user = next((u for u in o.project.users if u.jwt)) logger.info( "Refreshing observable project ({}/{}): observer:{} system:{} path:{}" .format(i, len(obs), importing_user, o.system_id, o.path)) current_user_names = set([u.username for u in o.project.users]) # we need to add any users who have been added to the system roles # (note that we do not delete any that are no longer listed on system roles; we only add users) system_users = set( get_system_users(o.project.tenant_id, importing_user.jwt, o.system_id)) updated_user_names = system_users.union(current_user_names) if updated_user_names != current_user_names: logger.info("Updating to add the following users:{} " "Updated user list is now: {}".format( updated_user_names - current_user_names, updated_user_names)) o.project.users = [ UserService.getOrCreateUser(u, tenant=o.project.tenant_id) for u in updated_user_names ] db_session.add(o) db_session.commit() # perform the importing if o.watch_content: import_from_agave(o.project.tenant_id, importing_user.id, o.system_id, o.path, o.project.id) except Exception: logger.exception( "Unhandled exception when importing observable project") db_session.rollback()
def create(data: dict, user: User) -> Project: """ Create a new map project for a user. :param data: dict :param user: User :return: Project """ project = Project(**data['project']) project.tenant_id = user.tenant_id project.users.append(user) if data.get('observable', False): try: ProjectsService.makeObservable( project, user, data.get('watch_content', False)) except Exception as e: logger.exception("{}".format(e)) raise e db_session.add(project) db_session.commit() return project
def task_fixture(): task = Task(process_id="1234", status="SUCCESS", description="description") db_session.add(task) db_session.commit() yield task
def import_from_agave(tenant_id: str, userId: int, systemId: str, path: str, projectId: int): user = db_session.query(User).get(userId) client = AgaveUtils(user.jwt) logger.info("Importing for project:{} directory:{}/{} for user:{}".format( projectId, systemId, path, user.username)) listing = client.listing(systemId, path) # First item is always a reference to self files_in_directory = listing[1:] filenames_in_directory = [str(f.path) for f in files_in_directory] for item in files_in_directory: if item.type == "dir" and not str(item.path).endswith("/.Trash"): import_from_agave(tenant_id, userId, systemId, item.path, projectId) # skip any junk files that are not allowed if item.path.suffix.lower().lstrip( '.') not in FeaturesService.ALLOWED_EXTENSIONS: continue else: try: # first check if there already is a file in the DB item_system_path = os.path.join(item.system, str(item.path).lstrip("/")) targetFile = ImportsService.getImport(projectId, systemId, str(item.path)) if targetFile: logger.info("Already imported {}".format(item_system_path)) continue # If its a RApp project folder, grab the metadata from tapis meta service if is_member_of_rapp_project_folder(item_system_path): logger.info("RApp: importing:{} for user:{}".format( item_system_path, user.username)) if item.path.suffix.lower().lstrip( '.' ) not in FeaturesService.ALLOWED_GEOSPATIAL_FEATURE_ASSET_EXTENSIONS: logger.info("{path} is unsupported; skipping.".format( path=item_system_path)) continue logger.info("{} {} {}".format(item_system_path, item.system, item.path)) try: meta = get_metadata_using_service_account( tenant_id, item.system, item.path) except MissingServiceAccount: logger.error( "No service account. Unable to get metadata for {}:{}" .format(item.system, item.path)) return {} logger.debug( "metadata from service account for file:{} : {}". format(item_system_path, meta)) if not meta: logger.info("No metadata for {}; skipping file".format( item_system_path)) continue geolocation = meta.get("geolocation") if not geolocation: logger.info("No geolocation for:{}; skipping".format( item_system_path)) continue lat, lon = _parse_rapid_geolocation(geolocation) tmpFile = client.getFile(systemId, item.path) feat = FeaturesService.fromLatLng(projectId, lat, lon, {}) feat.properties = meta db_session.add(feat) tmpFile.filename = Path(item.path).name try: FeaturesService.createFeatureAsset( projectId, feat.id, tmpFile, original_path=item_system_path) except: # remove newly-created placeholder feature if we fail to create an asset FeaturesService.delete(feat.id) raise RuntimeError("Unable to create feature asset") NotificationsService.create( user, "success", "Imported {f}".format(f=item_system_path)) tmpFile.close() elif item.path.suffix.lower().lstrip( '.') in FeaturesService.ALLOWED_GEOSPATIAL_EXTENSIONS: logger.info("importing:{} for user:{}".format( item_system_path, user.username)) tmpFile = client.getFile(systemId, item.path) tmpFile.filename = Path(item.path).name additional_files = get_additional_files( systemId, item.path, client, filenames_in_directory) FeaturesService.fromFileObj( projectId, tmpFile, {}, original_path=item_system_path, additional_files=additional_files) NotificationsService.create( user, "success", "Imported {f}".format(f=item_system_path)) tmpFile.close() else: continue # Save the row in the database that marks this file as already imported so it doesn't get added again targetFile = ImportsService.createImportedFile( projectId, systemId, str(item.path), item.lastModified) db_session.add(targetFile) db_session.commit() except Exception as e: db_session.rollback() logger.error( "Could not import for user:{} from agave:{}/{}".format( user.username, systemId, path)) NotificationsService.create( user, "error", "Error importing {f}".format(f=item_system_path)) logger.exception(e) continue
def import_point_clouds_from_agave(userId: int, files, pointCloudId: int): user = db_session.query(User).get(userId) client = AgaveUtils(user.jwt) point_cloud = pointcloud.PointCloudService.get(pointCloudId) celery_task_id = celery_uuid() task = Task() task.process_id = celery_task_id task.status = "RUNNING" point_cloud.task = task db_session.add(point_cloud) new_asset_files = [] failed_message = None for file in files: _update_point_cloud_task(pointCloudId, description="Importing file ({}/{})".format( len(new_asset_files) + 1, len(files))) NotificationsService.create(user, "success", task.description) system_id = file["system"] path = file["path"] try: tmp_file = client.getFile(system_id, path) tmp_file.filename = Path(path).name file_path = pointcloud.PointCloudService.putPointCloudInOriginalsFileDir( point_cloud.path, tmp_file, tmp_file.filename) tmp_file.close() # save file path as we might need to delete it if there is a problem new_asset_files.append(file_path) # check if file is okay check_point_cloud.apply(args=[file_path], throw=True) except InvalidCoordinateReferenceSystem: logger.error("Could not import point cloud file due to missing" " coordinate reference system: {}:{}".format( system_id, path)) failed_message = 'Error importing {}: missing coordinate reference system'.format( path) except Exception as e: logger.error( "Could not import point cloud file for user:{} from tapis: {}/{} : {}" .format(user.username, system_id, path, e)) failed_message = 'Unknown error importing {}:{}'.format( system_id, path) if failed_message: for file_path in new_asset_files: logger.info("removing {}".format(file_path)) os.remove(file_path) _update_point_cloud_task(pointCloudId, description=failed_message, status="FAILED") NotificationsService.create(user, "error", failed_message) return _update_point_cloud_task(pointCloudId, description="Running potree converter", status="RUNNING") point_cloud.files_info = json.dumps(get_point_cloud_info(pointCloudId)) try: db_session.add(point_cloud) db_session.add(task) db_session.commit() except: db_session.rollback() raise NotificationsService.create( user, "success", "Running potree converter (for point cloud {}).".format(pointCloudId)) try: convert_to_potree.apply(args=[pointCloudId], task_id=celery_task_id, throw=True) NotificationsService.create( user, "success", "Completed potree converter (for point cloud {}).".format( pointCloudId)) except: logger.exception("point cloud:{} conversion failed for user:{}".format( pointCloudId, user.username)) _update_point_cloud_task(pointCloudId, description="", status="FAILED") NotificationsService.create( user, "error", "Processing failed for point cloud ({})!".format(pointCloudId)) return
def public_projects_fixture(projects_fixture): projects_fixture.public = True db_session.add(projects_fixture) db_session.commit() yield projects_fixture
def convert_to_potree(self, pointCloudId: int) -> None: """ Use the potree converter to convert a LAS/LAZ file to potree format :param pointCloudId: int :return: None """ from geoapi.models import Feature, FeatureAsset from geoapi.services.point_cloud import PointCloudService point_cloud = PointCloudService.get(pointCloudId) path_to_original_point_clouds = get_asset_path( point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR) path_temp_processed_point_cloud_path = get_asset_path( point_cloud.path, PointCloudService.PROCESSED_DIR) input_files = [ get_asset_path(path_to_original_point_clouds, file) for file in os.listdir(path_to_original_point_clouds) if pathlib.Path(file).suffix.lstrip('.').lower() in PointCloudService.LIDAR_FILE_EXTENSIONS ] outline = get_bounding_box_2d(input_files) command = [ "PotreeConverter", "--verbose", "-i", path_to_original_point_clouds, "-o", path_temp_processed_point_cloud_path, "--overwrite", "--generate-page", "index" ] if point_cloud.conversion_parameters: command.extend(point_cloud.conversion_parameters.split()) logger.info("Processing point cloud (#{}): {}".format( pointCloudId, " ".join(command))) subprocess.run(command, check=True, capture_output=True, text=True) # Create preview viewer html (with no menu and now nsf logo) with open( os.path.join(path_temp_processed_point_cloud_path, "preview.html"), 'w+') as preview: with open( os.path.join(path_temp_processed_point_cloud_path, "index.html"), 'r') as viewer: content = viewer.read() content = re.sub(r"<div class=\"nsf_logo\"(.+?)</div>", '', content, flags=re.DOTALL) content = content.replace("viewer.toggleSidebar()", "$('.potree_menu_toggle').hide()") preview.write(content) if point_cloud.feature_id: feature = point_cloud.feature else: feature = Feature() feature.project_id = point_cloud.project_id asset_uuid = uuid.uuid4() base_filepath = make_project_asset_dir(point_cloud.project_id) asset_path = os.path.join(base_filepath, str(asset_uuid)) fa = FeatureAsset(uuid=asset_uuid, asset_type="point_cloud", path=get_asset_relative_path(asset_path), feature=feature) feature.assets.append(fa) point_cloud.feature = feature feature.the_geom = from_shape(geometries.convert_3D_2D(outline), srid=4326) point_cloud.task.status = "FINISHED" point_cloud.task.description = "" point_cloud_asset_path = get_asset_path(feature.assets[0].path) shutil.rmtree(point_cloud_asset_path, ignore_errors=True) shutil.move(path_temp_processed_point_cloud_path, point_cloud_asset_path) try: db_session.add(point_cloud) db_session.add(feature) db_session.commit() except: db_session.rollback() raise