def test_remove_feature_image(projects_fixture, image_file_fixture):
feature = FeaturesService.fromImage(projects_fixture.id, image_file_fixture, metadata={})
FeaturesService.delete(feature.id)
assert db_session.query(Feature).count() == 0
assert db_session.query(FeatureAsset).count() == 0
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 0
def test_remove_feature_image_asset(projects_fixture, feature_fixture, image_file_fixture):
feature = FeaturesService.createFeatureAsset(projects_fixture.id,
feature_fixture.id,
FileStorage(image_file_fixture))
FeaturesService.delete(feature.id)
assert db_session.query(Feature).count() == 0
assert db_session.query(FeatureAsset).count() == 0
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 0
def test_remove_tile_server(projects_fixture):
data = {
"name": "Test",
"type": "tms",
"url": "www.test.com",
"attribution": "contributors"
}
tile_server = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
FeaturesService.deleteTileServer(tile_server.id)
assert db_session.query(TileServer).count() == 0
def put(self, projectId: int, tileServerId: int):
u = request.current_user
logger.info("Update project:{} for user:{}".format(projectId,
u.username))
return FeaturesService.updateTileServer(tileServerId=tileServerId,
data=api.payload)
def test_insert_feature_geojson(projects_fixture, feature_properties_file_fixture):
features = FeaturesService.fromGeoJSON(projects_fixture.id, feature_properties_file_fixture, metadata={})
feature = features[0]
assert len(features) == 1
assert feature.project_id == projects_fixture.id
assert db_session.query(Feature).count() == 1
assert db_session.query(FeatureAsset).count() == 0
def put(self, projectId: int):
u = request.current_user
logger.info("Update project:{} for user:{}".format(projectId,
u.username))
ts = FeaturesService.updateTileServers(dataList=api.payload)
return ts
def post(self, projectId: int, featureId: int):
systemId = request.json["system_id"]
path = request.json["path"]
u = request.current_user
logger.info("Add feature asset to project:{} for user:{}: {}/{}".format(
projectId, u.username, systemId, path))
return FeaturesService.createFeatureAssetFromTapis(u, projectId, featureId, systemId, path)
def test_update_tile_server(projects_fixture):
data = {
"name": "Test",
"type": "tms",
"url": "www.test.com",
"attribution": "contributors"
}
FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
updated_data = {
"name": "NewTestName",
}
updated_tile_server = FeaturesService.updateTileServer(tileServerId=1,
data=updated_data)
assert updated_tile_server.name == "NewTestName"
def post(self, projectId: int):
file = request.files['file']
logger.info("Add feature to project:{} for user:{} : {}".format(
projectId, request.current_user.username, file.filename))
formData = request.form
metadata = formData.to_dict()
features = FeaturesService.fromFileObj(projectId, file, metadata)
return features
def test_delete_point_cloud_feature(celery_task_always_eager, projects_fixture,
point_cloud_fixture,
lidar_las1pt2_file_fixture):
PointCloudService.fromFileObj(point_cloud_fixture.id,
lidar_las1pt2_file_fixture,
lidar_las1pt2_file_fixture.name)
point_cloud = db_session.query(PointCloud).get(1)
feature_asset_path = get_asset_path(point_cloud.feature.assets[0].path)
FeaturesService.delete(point_cloud.feature.id)
assert db_session.query(PointCloud).count() == 1
assert db_session.query(PointCloud).get(1).feature is None
assert db_session.query(Feature).count() == 0
assert db_session.query(FeatureAsset).count() == 0
assert os.path.exists(
get_asset_path(point_cloud.path, PointCloudService.ORIGINAL_FILES_DIR))
assert not os.path.exists(feature_asset_path)
def test_update_tile_servers(projects_fixture):
data = {
"name": "Test",
"type": "tms",
"url": "www.test.com",
"attribution": "contributors"
}
resp1 = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
resp2 = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
updated_data = [{"id": resp1.id, "name": "NewTestName1"},
{"id": resp2.id, "name": "NewTestName2"}]
updated_tile_server_list = FeaturesService.updateTileServers(dataList=updated_data)
assert updated_tile_server_list[0].name == "NewTestName1"
assert updated_tile_server_list[1].name == "NewTestName2"
def test_create_feature_image_small_image(projects_fixture, image_small_DES_2176_fixture):
feature = FeaturesService.fromImage(projects_fixture.id, image_small_DES_2176_fixture, metadata={})
assert feature.project_id == projects_fixture.id
assert len(feature.assets) == 1
assert db_session.query(Feature).count() == 1
assert db_session.query(FeatureAsset).count() == 1
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 2
os.path.isfile(get_asset_path(feature.assets[0].path))
os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".thumb.jpeg"))
def test_create_feature_shpfile(projects_fixture, shapefile_fixture, shapefile_additional_files_fixture):
features = FeaturesService.fromShapefile(projects_fixture.id,
shapefile_fixture,
metadata={},
additional_files=shapefile_additional_files_fixture,
original_path="foo")
assert len(features) == 10
assert db_session.query(Feature).count() == 10
assert features[0].project_id == projects_fixture.id
def test_create_tile_server_from_file(projects_fixture, tile_server_ini_file_fixture):
tile_server = FeaturesService.fromINI(projects_fixture.id,
tile_server_ini_file_fixture,
metadata={})
assert tile_server.name == "Base OSM"
assert tile_server.type == "tms"
assert tile_server.url == "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
assert tile_server.attribution == "OpenStreetMap contributorshttps://www.openstreetmap.org/copyright"
def test_create_feature_video_asset(projects_fixture, feature_fixture, video_file_fixture):
feature = FeaturesService.createFeatureAsset(projects_fixture.id,
feature_fixture.id,
FileStorage(video_file_fixture))
assert feature.id == feature_fixture.id
assert len(feature.assets) == 1
assert db_session.query(FeatureAsset).count() == 1
assert len(os.listdir(get_project_asset_dir(feature.project_id))) == 1
os.path.isfile(get_asset_path(feature.assets[0].path))
os.path.isfile(os.path.join(get_project_asset_dir(projects_fixture.id), str(feature.assets[0].uuid) + ".mp4"))
def wrapper(*args, **kwargs):
projectId = kwargs.get("projectId")
proj = ProjectsService.get(projectId)
if not proj:
abort(404, "No project found")
featureId = kwargs.get("featureId")
feature = FeaturesService.get(featureId)
if not feature:
abort(404, "No feature found!")
if feature.project_id != projectId:
abort(404, "Feature not part of project")
return fn(*args, **kwargs)
def test_create_tile_server(projects_fixture):
data = {
"name": "Test",
"type": "tms",
"url": "www.test.com",
"attribution": "contributors"
}
tile_server = FeaturesService.addTileServer(projectId=projects_fixture.id, data=data)
assert tile_server.name == "Test"
assert tile_server.type == "tms"
assert tile_server.url == "www.test.com"
assert tile_server.attribution == "contributors"
def import_file_from_agave(userId: int, systemId: str, path: str,
projectId: int):
user = db_session.query(User).get(userId)
client = AgaveUtils(user.jwt)
try:
tmpFile = client.getFile(systemId, path)
tmpFile.filename = Path(path).name
additional_files = get_additional_files(systemId, path, client)
FeaturesService.fromFileObj(projectId,
tmpFile, {},
original_path=path,
additional_files=additional_files)
NotificationsService.create(user, "success",
"Imported {f}".format(f=path))
tmpFile.close()
except Exception as e:
db_session.rollback()
logger.exception("Could not import file from agave: {} :: {}".format(
systemId, path))
NotificationsService.create(user, "error",
"Error importing {f}".format(f=path))
raise e
def post(self, projectId: int):
u = request.current_user
logger.info("Import overlay to project:{} for user:{} : {}".format(
projectId, u.username, request.json))
systemId = request.json['system_id']
path = request.json['path']
label = request.json['label']
bounds = [
request.json['minLon'],
request.json['minLat'],
request.json['maxLon'],
request.json['maxLat']
]
ov = FeaturesService.addOverlayFromTapis(u, projectId, systemId, path, bounds, label)
return ov
def post(self, projectId: int):
file = request.files['file']
logger.info("Add overlay to project:{} for user:{} : {}".format(
projectId, request.current_user.username, file.filename))
formData = request.form
bounds = [
formData['minLon'],
formData['minLat'],
formData['maxLon'],
formData['maxLat']
]
label = formData['label']
ov = FeaturesService.addOverlay(projectId, file, bounds, label)
return ov
def post(self, projectId: int):
logger.info("Add tile server to project:{} for user:{}".format(
projectId, request.current_user.username))
ts = FeaturesService.addTileServer(projectId, api.payload)
return ts
def delete(self, projectId: int, overlayId: int) -> str:
logger.info("Delete overlay:{} in project:{} for user:{}".format(
overlayId, projectId, request.current_user.username))
FeaturesService.deleteOverlay(projectId, overlayId)
return "Overlay {id} deleted".format(id=overlayId)
def test_create_feature_fromLatLng(projects_fixture):
feature = FeaturesService.fromLatLng(projects_fixture.id, 10, 20, metadata={})
assert len(feature.assets) == 0
assert feature.id is not None
def get(self, projectId: int):
ovs = FeaturesService.getOverlays(projectId)
return ovs
def test_hazmapperv1_file_with_images(projects_fixture, hazmpperV1_file):
features = FeaturesService.fromGeoJSON(projects_fixture.id, hazmpperV1_file, metadata={})
assert len(features) == 2
assert len(features[1].assets) == 1
def test_remove_feature(projects_fixture, feature_fixture):
FeaturesService.delete(feature_fixture.id)
assert db_session.query(Feature).count() == 0
assert not os.path.exists(get_project_asset_dir(feature_fixture.project_id))
def delete(self, projectId: int, tileServerId: int) -> str:
logger.info("Delete tile server:{} in project:{} for user:{}".format(
tileServerId, projectId, request.current_user.username))
FeaturesService.deleteTileServer(tileServerId)
return "Tile Server {id} deleted".format(id=tileServerId)
def import_from_agave(tenant_id: str, userId: int, systemId: str, path: str,
projectId: int):
user = db_session.query(User).get(userId)
client = AgaveUtils(user.jwt)
logger.info("Importing for project:{} directory:{}/{} for user:{}".format(
projectId, systemId, path, user.username))
listing = client.listing(systemId, path)
# First item is always a reference to self
files_in_directory = listing[1:]
filenames_in_directory = [str(f.path) for f in files_in_directory]
for item in files_in_directory:
if item.type == "dir" and not str(item.path).endswith("/.Trash"):
import_from_agave(tenant_id, userId, systemId, item.path,
projectId)
# skip any junk files that are not allowed
if item.path.suffix.lower().lstrip(
'.') not in FeaturesService.ALLOWED_EXTENSIONS:
continue
else:
try:
# first check if there already is a file in the DB
item_system_path = os.path.join(item.system,
str(item.path).lstrip("/"))
targetFile = ImportsService.getImport(projectId, systemId,
str(item.path))
if targetFile:
logger.info("Already imported {}".format(item_system_path))
continue
# If its a RApp project folder, grab the metadata from tapis meta service
if is_member_of_rapp_project_folder(item_system_path):
logger.info("RApp: importing:{} for user:{}".format(
item_system_path, user.username))
if item.path.suffix.lower().lstrip(
'.'
) not in FeaturesService.ALLOWED_GEOSPATIAL_FEATURE_ASSET_EXTENSIONS:
logger.info("{path} is unsupported; skipping.".format(
path=item_system_path))
continue
logger.info("{} {} {}".format(item_system_path,
item.system, item.path))
try:
meta = get_metadata_using_service_account(
tenant_id, item.system, item.path)
except MissingServiceAccount:
logger.error(
"No service account. Unable to get metadata for {}:{}"
.format(item.system, item.path))
return {}
logger.debug(
"metadata from service account for file:{} : {}".
format(item_system_path, meta))
if not meta:
logger.info("No metadata for {}; skipping file".format(
item_system_path))
continue
geolocation = meta.get("geolocation")
if not geolocation:
logger.info("No geolocation for:{}; skipping".format(
item_system_path))
continue
lat, lon = _parse_rapid_geolocation(geolocation)
tmpFile = client.getFile(systemId, item.path)
feat = FeaturesService.fromLatLng(projectId, lat, lon, {})
feat.properties = meta
db_session.add(feat)
tmpFile.filename = Path(item.path).name
try:
FeaturesService.createFeatureAsset(
projectId,
feat.id,
tmpFile,
original_path=item_system_path)
except:
# remove newly-created placeholder feature if we fail to create an asset
FeaturesService.delete(feat.id)
raise RuntimeError("Unable to create feature asset")
NotificationsService.create(
user, "success",
"Imported {f}".format(f=item_system_path))
tmpFile.close()
elif item.path.suffix.lower().lstrip(
'.') in FeaturesService.ALLOWED_GEOSPATIAL_EXTENSIONS:
logger.info("importing:{} for user:{}".format(
item_system_path, user.username))
tmpFile = client.getFile(systemId, item.path)
tmpFile.filename = Path(item.path).name
additional_files = get_additional_files(
systemId, item.path, client, filenames_in_directory)
FeaturesService.fromFileObj(
projectId,
tmpFile, {},
original_path=item_system_path,
additional_files=additional_files)
NotificationsService.create(
user, "success",
"Imported {f}".format(f=item_system_path))
tmpFile.close()
else:
continue
# Save the row in the database that marks this file as already imported so it doesn't get added again
targetFile = ImportsService.createImportedFile(
projectId, systemId, str(item.path), item.lastModified)
db_session.add(targetFile)
db_session.commit()
except Exception as e:
db_session.rollback()
logger.error(
"Could not import for user:{} from agave:{}/{}".format(
user.username, systemId, path))
NotificationsService.create(
user, "error",
"Error importing {f}".format(f=item_system_path))
logger.exception(e)
continue
def test_insert_feature_collection(projects_fixture, geojson_file_fixture):
features = FeaturesService.fromGeoJSON(projects_fixture.id, geojson_file_fixture, metadata={})
for feature in features:
assert feature.project_id == projects_fixture.id
assert db_session.query(Feature).count() == 3
assert db_session.query(FeatureAsset).count() == 0
def get(self, projectId: int):
tsv = FeaturesService.getTileServers(projectId)
return tsv