def feature_fixture():
home = os.path.dirname(__file__)
with open(os.path.join(home, 'fixtures/properties.json'), 'rb') as f:
feat = Feature.fromGeoJSON(json.loads(f.read()))
feat.project_id = 1
db_session.add(feat)
db_session.commit()
yield feat
def on_failure(self, exc, task_id, args, kwargs, einfo):
logger.info("Task ({}, point cloud {}) failed: {}".format(
task_id, args, exc))
failed_task = db_session.query(Task).filter(
Task.process_id == task_id).first()
failed_task.status = "FAILED"
failed_task.description = ""
db_session.add(failed_task)
db_session.commit()
def updateTileServers(dataList: List[dict]):
ret_list = []
for tsv in dataList:
ts = db_session.query(TileServer).get(int(tsv['id']))
for key, value in tsv.items():
setattr(ts, key, value)
ret_list.append(ts)
db_session.commit()
return ret_list
def projects_fixture2():
proj = Project(name="test2", description="description2")
u1 = db_session.query(User).filter(User.username == "test1").first()
proj.users.append(u1)
proj.tenant_id = u1.tenant_id
db_session.add(proj)
db_session.commit()
yield proj
shutil.rmtree(get_project_asset_dir(proj.id), ignore_errors=True)
def fromLatLng(projectId: int, lat: float, lng: float,
metadata: Dict) -> Feature:
point = Point(lng, lat)
f = Feature()
f.project_id = projectId
f.the_geom = from_shape(point, srid=4326)
f.properties = metadata or {}
db_session.add(f)
db_session.commit()
return f
def delete(pointCloudId: int) -> None:
"""
Delete a PointCloud
:param pointCloudId: int
:return: None
"""
point_cloud = PointCloudService.get(pointCloudId)
delete_assets(projectId=point_cloud.project_id, uuid=point_cloud.uuid)
db_session.delete(point_cloud)
db_session.commit()
def create(username: str, tenant: str, jwt: str = None) -> User:
"""
:rtype: User
"""
u = User(username=username, tenant_id=tenant)
if jwt:
u.jwt = jwt
db_session.add(u)
db_session.commit()
return u
def setStyles(featureId: int, styles: Dict) -> Feature:
"""
Set the styles of a feature.
:param featureId: int
:param styles: dict
:return: Feature
"""
feat = db_session.query(Feature).get(featureId)
# TODO: PROTECT assets and styles attributes
feat.styles = styles
db_session.commit()
return feat
def create(user: User, status: AnyStr, message: AnyStr) -> Notification:
note = Notification(username=user.username,
tenant_id=user.tenant_id,
status=status,
message=message)
try:
db_session.add(note)
db_session.commit()
return note
except Exception:
db_session.rollback()
raise
def delete(featureId: int) -> None:
"""
Delete a Feature and any assets tied to it.
:param featureId: int
:return: None
"""
feat = db_session.query(Feature).get(featureId)
assets = db_session.query(FeatureAsset).filter(
FeatureAsset.feature_id == featureId)
for asset in assets:
delete_assets(projectId=feat.project_id, uuid=asset.uuid)
db_session.delete(feat)
db_session.commit()
def addRandomMarkers():
proj = Project(name="test", description="test", tenant_id="designsafe")
user = db_session.query(User).filter(User.username == "jmeiring").first()
proj.users.append(user)
db_session.add(user)
for i in range(0, 10000):
p = Point(random.uniform(-180, 180), random.uniform(-90, 90))
feat = Feature(
the_geom=from_shape(p, srid=4326),
)
feat.project = proj
db_session.add(feat)
db_session.commit()
def setProperties(featureId: int, props: Dict) -> Feature:
"""
Set the properties of a feature.
:param featureId: int
:param props: dict
:return: Feature
"""
feat = db_session.query(Feature).get(featureId)
# TODO: Throw assert if not found?
# TODO: PROTECT assets and styles attributes
feat.properties = props
db_session.commit()
return feat
def addUserToProject(projectId: int, username: str) -> None:
"""
Add a user to a project
:param projectId: int
:param username: string
:return:
"""
# TODO: Add TAS integration
proj = db_session.query(Project).get(projectId)
user = UserService.getOrCreateUser(username, proj.tenant_id)
proj.users.append(user)
db_session.commit()
def _update_point_cloud_task(pointCloudId: int,
description: str = None,
status: str = None):
task = pointcloud.PointCloudService.get(pointCloudId).task
if description is not None:
task.description = description
if status is not None:
task.status = status
try:
db_session.add(task)
db_session.commit()
except Exception:
db_session.rollback()
raise
def fromImage(projectId: int,
fileObj: IO,
metadata: Dict,
original_path: str = None) -> Feature:
"""
Create a Point feature from a georeferenced image
:param projectId: int
:param fileObj: file
:param metadata: dict
:return: None
"""
imdata = ImageService.processImage(fileObj)
point = Point(imdata.coordinates)
f = Feature()
f.project_id = projectId
f.the_geom = from_shape(point, srid=4326)
f.properties = metadata
asset_uuid = uuid.uuid4()
base_filepath = make_project_asset_dir(projectId)
asset_path = os.path.join(base_filepath, str(asset_uuid) + '.jpeg')
fa = FeatureAsset(
uuid=asset_uuid,
asset_type="image",
original_path=original_path,
display_path=original_path,
path=get_asset_relative_path(asset_path),
feature=f,
)
f.assets.append(fa)
thumbnail_path = os.path.join(base_filepath,
str(asset_uuid) + ".thumb.jpeg")
resized_image_path = os.path.join(base_filepath,
str(asset_uuid) + '.jpeg')
try:
imdata.thumb.save(thumbnail_path, "JPEG")
imdata.resized.save(resized_image_path, "JPEG")
except:
if os.path.exists(thumbnail_path):
os.remove(thumbnail_path)
if os.path.exists(resized_image_path):
os.remove(resized_image_path)
raise
finally:
fileObj.close()
db_session.add(f)
db_session.commit()
return f
def makeObservable(proj: Project, user: User, watch_content: bool):
"""
Makes a project an observable project
Requires project's system_path, system_id, tenant_id to exist
:param proj: Project
:param user: User
:param watch_content: bool
:return: None
"""
folder_name = Path(proj.system_path).name
name = proj.system_id + '/' + folder_name
# TODO: Handle no storage system found
system = AgaveUtils(user.jwt).systemsGet(proj.system_id)
obs = ObservableDataProject(system_id=proj.system_id,
path=proj.system_path,
watch_content=watch_content)
users = get_system_users(proj.tenant_id, user.jwt, proj.system_id)
logger.info(
"Updating project:{} to have the following users: {}".format(
name, users))
project_users = [
UserService.getOrCreateUser(u, tenant=proj.tenant_id)
for u in users
]
proj.users = project_users
obs.project = proj
try:
db_session.add(obs)
db_session.commit()
except IntegrityError as e:
db_session.rollback()
logger.exception(
"User:{} tried to create an observable project that already exists: '{}'"
.format(user.username, name))
raise ObservableProjectAlreadyExists(
"'{}' project already exists".format(name))
if watch_content:
import_from_agave.apply_async(args=[
obs.project.tenant_id, user.id, obs.system_id, obs.path,
obs.project_id
])
def addTileServer(projectId: int, data: Dict):
"""
:param projectId: int
:param data: Dict
:return: ts: TileServer
"""
ts = TileServer()
for key, value in data.items():
setattr(ts, key, value)
ts.project_id = projectId
db_session.add(ts)
db_session.commit()
return ts
def delete(user: User, projectId: int) -> dict:
"""
Delete a project and all its Features and assets
:param projectId: int
:return:
"""
proj = db_session.query(Project).get(projectId)
db_session.delete(proj)
db_session.commit()
assets_folder = get_project_asset_dir(projectId)
try:
shutil.rmtree(assets_folder)
except FileNotFoundError:
pass
return {"status": "ok"}
def update(user: User, projectId: int, data: dict) -> Project:
"""
Update the metadata associated with a project
:param projectId: int
:param data: dict
:return: Project
"""
proj = ProjectsService.get(project_id=projectId)
proj_data = data.get('project', {})
proj.name = proj_data.get('name', proj.name)
proj.description = proj_data.get('description', proj.description)
proj.public = proj_data.get('public', proj.public)
db_session.commit()
return proj
def observable_projects_fixture():
u1 = db_session.query(User).filter(User.username == "test1").first()
proj = Project(name="test_observable",
description="description",
tenant_id=u1.tenant_id)
obs = ObservableDataProject(
system_id="testSystem",
path="/testPath",
watch_content=True
)
obs.project = proj
proj.users.append(u1)
db_session.add(obs)
db_session.add(proj)
db_session.commit()
yield obs
shutil.rmtree(get_project_asset_dir(proj.id), ignore_errors=True)
def update(pointCloudId: int, data: dict) -> PointCloud:
"""
Update a PointCloud
:param pointCloudId: int
:param data: dict
:return: Project
"""
point_cloud = PointCloudService.get(pointCloudId)
previous_conversion_parameters = point_cloud.conversion_parameters
for key, value in data.items():
setattr(point_cloud, key, value)
db_session.commit()
if 'conversion_parameters' in data and previous_conversion_parameters != data['conversion_parameters']:
PointCloudService._process_point_clouds(pointCloudId)
return point_cloud
def fromGPX(projectId: int,
fileObj: IO,
metadata: Dict,
original_path=None) -> Feature:
# TODO: Fiona should support reading from the file directly, this MemoryFile business
# should not be needed
with fiona.io.MemoryFile(fileObj) as memfile:
with memfile.open(layer="tracks") as collection:
track = collection[0]
shp = shape(track["geometry"])
feat = Feature()
feat.project_id = projectId
feat.the_geom = from_shape(geometries.convert_3D_2D(shp),
srid=4326)
feat.properties = metadata or {}
db_session.add(feat)
db_session.commit()
return feat
def create(projectId: int, data: dict, user: User) -> PointCloud:
"""
Create a PointCloud for a user.
:param projectId: int
:param data: dict
:param user: User
:return: PointCloud
"""
point_cloud_uid = uuid.uuid4()
point_cloud_path = os.path.join(make_project_asset_dir(projectId), str(point_cloud_uid))
file_point_cloud_path = os.path.join(point_cloud_path, PointCloudService.ORIGINAL_FILES_DIR)
pathlib.Path(file_point_cloud_path).mkdir(parents=True, exist_ok=True)
point_cloud = PointCloud(**data)
point_cloud.project_id = projectId
point_cloud.tenant_id = user.tenant_id
point_cloud.uuid = point_cloud_uid
point_cloud.path = get_asset_relative_path(point_cloud_path)
db_session.add(point_cloud)
db_session.commit()
return point_cloud
def refresh_observable_projects():
try:
obs = db_session.query(ObservableDataProject).all()
for i, o in enumerate(obs):
# we need a user with a jwt for importing
importing_user = next((u for u in o.project.users if u.jwt))
logger.info(
"Refreshing observable project ({}/{}): observer:{} system:{} path:{}"
.format(i, len(obs), importing_user, o.system_id, o.path))
current_user_names = set([u.username for u in o.project.users])
# we need to add any users who have been added to the system roles
# (note that we do not delete any that are no longer listed on system roles; we only add users)
system_users = set(
get_system_users(o.project.tenant_id, importing_user.jwt,
o.system_id))
updated_user_names = system_users.union(current_user_names)
if updated_user_names != current_user_names:
logger.info("Updating to add the following users:{} "
"Updated user list is now: {}".format(
updated_user_names - current_user_names,
updated_user_names))
o.project.users = [
UserService.getOrCreateUser(u, tenant=o.project.tenant_id)
for u in updated_user_names
]
db_session.add(o)
db_session.commit()
# perform the importing
if o.watch_content:
import_from_agave(o.project.tenant_id, importing_user.id,
o.system_id, o.path, o.project.id)
except Exception:
logger.exception(
"Unhandled exception when importing observable project")
db_session.rollback()
def create(data: dict, user: User) -> Project:
"""
Create a new map project for a user.
:param data: dict
:param user: User
:return: Project
"""
project = Project(**data['project'])
project.tenant_id = user.tenant_id
project.users.append(user)
if data.get('observable', False):
try:
ProjectsService.makeObservable(
project, user, data.get('watch_content', False))
except Exception as e:
logger.exception("{}".format(e))
raise e
db_session.add(project)
db_session.commit()
return project
def deleteOverlay(projectId: int, overlayId: int) -> None:
ov = db_session.query(Overlay).get(overlayId)
delete_assets(projectId=projectId, uuid=ov.uuid)
db_session.delete(ov)
db_session.commit()
def import_from_agave(tenant_id: str, userId: int, systemId: str, path: str,
projectId: int):
user = db_session.query(User).get(userId)
client = AgaveUtils(user.jwt)
logger.info("Importing for project:{} directory:{}/{} for user:{}".format(
projectId, systemId, path, user.username))
listing = client.listing(systemId, path)
# First item is always a reference to self
files_in_directory = listing[1:]
filenames_in_directory = [str(f.path) for f in files_in_directory]
for item in files_in_directory:
if item.type == "dir" and not str(item.path).endswith("/.Trash"):
import_from_agave(tenant_id, userId, systemId, item.path,
projectId)
# skip any junk files that are not allowed
if item.path.suffix.lower().lstrip(
'.') not in FeaturesService.ALLOWED_EXTENSIONS:
continue
else:
try:
# first check if there already is a file in the DB
item_system_path = os.path.join(item.system,
str(item.path).lstrip("/"))
targetFile = ImportsService.getImport(projectId, systemId,
str(item.path))
if targetFile:
logger.info("Already imported {}".format(item_system_path))
continue
# If its a RApp project folder, grab the metadata from tapis meta service
if is_member_of_rapp_project_folder(item_system_path):
logger.info("RApp: importing:{} for user:{}".format(
item_system_path, user.username))
if item.path.suffix.lower().lstrip(
'.'
) not in FeaturesService.ALLOWED_GEOSPATIAL_FEATURE_ASSET_EXTENSIONS:
logger.info("{path} is unsupported; skipping.".format(
path=item_system_path))
continue
logger.info("{} {} {}".format(item_system_path,
item.system, item.path))
try:
meta = get_metadata_using_service_account(
tenant_id, item.system, item.path)
except MissingServiceAccount:
logger.error(
"No service account. Unable to get metadata for {}:{}"
.format(item.system, item.path))
return {}
logger.debug(
"metadata from service account for file:{} : {}".
format(item_system_path, meta))
if not meta:
logger.info("No metadata for {}; skipping file".format(
item_system_path))
continue
geolocation = meta.get("geolocation")
if not geolocation:
logger.info("No geolocation for:{}; skipping".format(
item_system_path))
continue
lat, lon = _parse_rapid_geolocation(geolocation)
tmpFile = client.getFile(systemId, item.path)
feat = FeaturesService.fromLatLng(projectId, lat, lon, {})
feat.properties = meta
db_session.add(feat)
tmpFile.filename = Path(item.path).name
try:
FeaturesService.createFeatureAsset(
projectId,
feat.id,
tmpFile,
original_path=item_system_path)
except:
# remove newly-created placeholder feature if we fail to create an asset
FeaturesService.delete(feat.id)
raise RuntimeError("Unable to create feature asset")
NotificationsService.create(
user, "success",
"Imported {f}".format(f=item_system_path))
tmpFile.close()
elif item.path.suffix.lower().lstrip(
'.') in FeaturesService.ALLOWED_GEOSPATIAL_EXTENSIONS:
logger.info("importing:{} for user:{}".format(
item_system_path, user.username))
tmpFile = client.getFile(systemId, item.path)
tmpFile.filename = Path(item.path).name
additional_files = get_additional_files(
systemId, item.path, client, filenames_in_directory)
FeaturesService.fromFileObj(
projectId,
tmpFile, {},
original_path=item_system_path,
additional_files=additional_files)
NotificationsService.create(
user, "success",
"Imported {f}".format(f=item_system_path))
tmpFile.close()
else:
continue
# Save the row in the database that marks this file as already imported so it doesn't get added again
targetFile = ImportsService.createImportedFile(
projectId, systemId, str(item.path), item.lastModified)
db_session.add(targetFile)
db_session.commit()
except Exception as e:
db_session.rollback()
logger.error(
"Could not import for user:{} from agave:{}/{}".format(
user.username, systemId, path))
NotificationsService.create(
user, "error",
"Error importing {f}".format(f=item_system_path))
logger.exception(e)
continue
def import_point_clouds_from_agave(userId: int, files, pointCloudId: int):
user = db_session.query(User).get(userId)
client = AgaveUtils(user.jwt)
point_cloud = pointcloud.PointCloudService.get(pointCloudId)
celery_task_id = celery_uuid()
task = Task()
task.process_id = celery_task_id
task.status = "RUNNING"
point_cloud.task = task
db_session.add(point_cloud)
new_asset_files = []
failed_message = None
for file in files:
_update_point_cloud_task(pointCloudId,
description="Importing file ({}/{})".format(
len(new_asset_files) + 1, len(files)))
NotificationsService.create(user, "success", task.description)
system_id = file["system"]
path = file["path"]
try:
tmp_file = client.getFile(system_id, path)
tmp_file.filename = Path(path).name
file_path = pointcloud.PointCloudService.putPointCloudInOriginalsFileDir(
point_cloud.path, tmp_file, tmp_file.filename)
tmp_file.close()
# save file path as we might need to delete it if there is a problem
new_asset_files.append(file_path)
# check if file is okay
check_point_cloud.apply(args=[file_path], throw=True)
except InvalidCoordinateReferenceSystem:
logger.error("Could not import point cloud file due to missing"
" coordinate reference system: {}:{}".format(
system_id, path))
failed_message = 'Error importing {}: missing coordinate reference system'.format(
path)
except Exception as e:
logger.error(
"Could not import point cloud file for user:{} from tapis: {}/{} : {}"
.format(user.username, system_id, path, e))
failed_message = 'Unknown error importing {}:{}'.format(
system_id, path)
if failed_message:
for file_path in new_asset_files:
logger.info("removing {}".format(file_path))
os.remove(file_path)
_update_point_cloud_task(pointCloudId,
description=failed_message,
status="FAILED")
NotificationsService.create(user, "error", failed_message)
return
_update_point_cloud_task(pointCloudId,
description="Running potree converter",
status="RUNNING")
point_cloud.files_info = json.dumps(get_point_cloud_info(pointCloudId))
try:
db_session.add(point_cloud)
db_session.add(task)
db_session.commit()
except:
db_session.rollback()
raise
NotificationsService.create(
user, "success",
"Running potree converter (for point cloud {}).".format(pointCloudId))
try:
convert_to_potree.apply(args=[pointCloudId],
task_id=celery_task_id,
throw=True)
NotificationsService.create(
user, "success",
"Completed potree converter (for point cloud {}).".format(
pointCloudId))
except:
logger.exception("point cloud:{} conversion failed for user:{}".format(
pointCloudId, user.username))
_update_point_cloud_task(pointCloudId, description="", status="FAILED")
NotificationsService.create(
user, "error",
"Processing failed for point cloud ({})!".format(pointCloudId))
return
def deleteTileServer(tileServerId: int) -> None:
ts = db_session.query(TileServer).get(tileServerId)
db_session.delete(ts)
db_session.commit()
def updateTileServer(tileServerId: int, data: dict):
ts = db_session.query(TileServer).get(tileServerId)
for key, value in data.items():
setattr(ts, key, value)
db_session.commit()
return ts
