def create_canned_project() -> Tuple[Project, User]:
""" Generates a canned project in the DB to help with integration tests """
test_aoi_geojson = geojson.loads(json.dumps(get_canned_json("test_aoi.json")))
task_feature = geojson.loads(json.dumps(get_canned_json("splittable_task.json")))
task_non_square_feature = geojson.loads(
json.dumps(get_canned_json("non_square_task.json"))
)
test_user = create_canned_user()
test_project_dto = DraftProjectDTO()
test_project_dto.project_name = "Test"
test_project_dto.user_id = test_user.id
test_project_dto.area_of_interest = test_aoi_geojson
test_project = Project()
test_project.create_draft_project(test_project_dto)
test_project.set_project_aoi(test_project_dto)
test_project.total_tasks = 2
# Setup test task
test_task = Task.from_geojson_feature(1, task_feature)
test_task.task_status = TaskStatus.MAPPED.value
test_task.mapped_by = test_user.id
test_task.is_square = True
test_task2 = Task.from_geojson_feature(2, task_non_square_feature)
test_task2.task_status = TaskStatus.READY.value
test_task2.is_square = False
test_project.tasks.append(test_task)
test_project.tasks.append(test_task2)
test_project.create()
return test_project, test_user
def test_split_geom_raise_grid_service_error_when_task_not_usable(self):
if self.skip_tests:
return
with self.assertRaises(SplitServiceError):
task_stub = Task()
task_stub.is_square = True
SplitService._create_split_tasks("foo", "bar", "dum", task_stub)
def test_cant_add_task_if_feature_geometry_is_invalid(self):
# Arrange
invalid_feature = geojson.loads(
'{"geometry": {"coordinates": [[[[-4.0237, 56.0904], [-3.9111, 56.1715],'
'[-3.8122, 56.098], [-4.0237]]]], "type": "MultiPolygon"}, "properties":'
'{"x": 2402, "y": 1736, "zoom": 12}, "type": "Feature"}')
with self.assertRaises(InvalidGeoJson):
Task.from_geojson_feature(1, invalid_feature)
def test_cant_add_task_if_feature_has_missing_properties(self):
# Missing zoom
invalid_properties = geojson.loads(
'{"geometry": {"coordinates": [[[[-4.0237, 56.0904], [-3.9111, 56.1715],'
'[-3.8122, 56.098], [-4.0237, 56.0904]]]], "type": "MultiPolygon"},'
'"properties": {"x": 2402, "y": 1736}, "type": "Feature"}')
with self.assertRaises(InvalidData):
Task.from_geojson_feature(1, invalid_properties)
def test_cant_add_task_if_not_supplied_feature_type(self):
# Arrange
invalid_feature = geojson.MultiPolygon([[(2.38, 57.322),
(23.194, -20.28),
(-120.43, 19.15),
(2.38, 10.33)]])
# Arrange
with self.assertRaises(InvalidGeoJson):
Task.from_geojson_feature(1, invalid_feature)
def test_lock_task_for_mapping_adds_locked_history(self):
# Arrange
test_task = Task()
# Act
test_task.set_task_history(action=TaskAction.LOCKED_FOR_MAPPING,
user_id=123454)
# Assert
self.assertEqual(TaskAction.LOCKED_FOR_MAPPING.name,
test_task.task_history[0].action)
def test_reset_task_clears_any_existing_locks(self, mock_set_task_history,
mock_record_auto_unlock,
mock_update):
user_id = 123
test_task = Task()
test_task.task_status = TaskStatus.LOCKED_FOR_MAPPING
test_task.reset_task(user_id)
mock_record_auto_unlock.assert_called()
self.assertEqual(test_task.task_status, TaskStatus.READY.value)
def test_reset_task_sets_to_ready_status(self, mock_set_task_history,
mock_update):
user_id = 123
test_task = Task()
test_task.task_status = TaskStatus.MAPPED
test_task.reset_task(user_id)
mock_set_task_history.assert_called_with(TaskAction.STATE_CHANGE,
user_id, None,
TaskStatus.READY)
mock_update.assert_called()
self.assertEqual(test_task.task_status, TaskStatus.READY.value)
def auto_unlock_tasks():
with application.app_context():
# Identify distinct project IDs that were touched in the last 2 hours
query = (TaskHistory.query.with_entities(
TaskHistory.project_id
).filter(
func.DATE(TaskHistory.action_date) > datetime.datetime.utcnow() -
datetime.timedelta(minutes=130)).distinct())
projects = query.all()
# For each project update task history for tasks that were not manually unlocked
for project in projects:
project_id = project[0]
Task.auto_unlock_tasks(project_id)
def generate_osm_xml(project_id: int, task_ids_str: str) -> str:
""" Generate xml response suitable for loading into JOSM. A sample output file is in
/backend/helpers/testfiles/osm-sample.xml """
# Note XML created with upload No to ensure it will be rejected by OSM if uploaded by mistake
root = ET.Element(
"osm",
attrib=dict(version="0.6",
upload="never",
creator="HOT Tasking Manager"),
)
if task_ids_str:
task_ids = map(int, task_ids_str.split(","))
tasks = Task.get_tasks(project_id, task_ids)
if not tasks or len(tasks) == 0:
raise NotFound()
else:
tasks = Task.get_all_tasks(project_id)
if not tasks or len(tasks) == 0:
raise NotFound()
fake_id = -1 # We use fake-ids to ensure XML will not be validated by OSM
for task in tasks:
task_geom = shape.to_shape(task.geometry)
way = ET.SubElement(
root,
"way",
attrib=dict(id=str((task.id * -1)),
action="modify",
visible="true"),
)
for poly in task_geom:
for point in poly.exterior.coords:
ET.SubElement(
root,
"node",
attrib=dict(
action="modify",
visible="true",
id=str(fake_id),
lon=str(point[0]),
lat=str(point[1]),
),
)
ET.SubElement(way, "nd", attrib=dict(ref=str(fake_id)))
fake_id -= 1
xml_gpx = ET.tostring(root, encoding="utf8")
return xml_gpx
def test_split_geom_returns_split_geometries(self):
# arrange
x = 2021
y = 2798
zoom = 12
task_stub = Task()
task_stub.is_square = True
expected = geojson.loads(json.dumps(get_canned_json("split_task.json")))
# act
result = SplitService._create_split_tasks(x, y, zoom, task_stub)
# assert
self.assertEqual(str(expected), str(result))
def as_dto_for_mapping(
self, authenticated_user_id: int = None, locale: str = "en", abbrev: bool = True
) -> Optional[ProjectDTO]:
""" Creates a Project DTO suitable for transmitting to mapper users """
# Check for project visibility settings
is_allowed_user = True
if self.status == ProjectStatus.DRAFT.value:
if not self.check_draft_project_visibility(authenticated_user_id):
is_allowed_user = False
if self.private:
is_allowed_user = False
if authenticated_user_id:
user = User.get_by_id(authenticated_user_id)
if (
UserRole(user.role) == UserRole.ADMIN
or authenticated_user_id == self.author_id
):
is_allowed_user = True
for user in self.allowed_users:
if user.id == authenticated_user_id:
is_allowed_user = True
break
if is_allowed_user:
project, project_dto = self._get_project_and_base_dto()
if abbrev is False:
project_dto.tasks = Task.get_tasks_as_geojson_feature_collection(
self.id, None
)
else:
project_dto.tasks = Task.get_tasks_as_geojson_feature_collection_no_geom(
self.id
)
project_dto.project_info = ProjectInfo.get_dto_for_locale(
self.id, locale, project.default_locale
)
if project.organisation_id:
project_dto.organisation = project.organisation.id
project_dto.organisation_name = project.organisation.name
project_dto.organisation_logo = project.organisation.logo
project_dto.project_info_locales = ProjectInfo.get_dto_for_all_locales(
self.id
)
return project_dto
def tasks_as_geojson(
self, task_ids_str: str, order_by=None, order_by_type="ASC", status=None
):
""" Creates a geojson of all areas """
project_tasks = Task.get_tasks_as_geojson_feature_collection(
self.id, task_ids_str, order_by, order_by_type, status
)
return project_tasks
def lock_tasks_for_validation(
validation_dto: LockForValidationDTO) -> TaskDTOs:
"""
Lock supplied tasks for validation
:raises ValidatorServiceError
"""
# Loop supplied tasks to check they can all be locked for validation
tasks_to_lock = []
for task_id in validation_dto.task_ids:
task = Task.get(task_id, validation_dto.project_id)
if task is None:
raise NotFound(f"Task {task_id} not found")
if TaskStatus(task.task_status) not in [
TaskStatus.MAPPED,
TaskStatus.INVALIDATED,
TaskStatus.BADIMAGERY,
]:
raise ValidatorServiceError(
f"Task {task_id} is not MAPPED, BADIMAGERY or INVALIDATED")
user_can_validate = ValidatorService._user_can_validate_task(
validation_dto.user_id, task.mapped_by)
if not user_can_validate:
raise ValidatorServiceError(
"Tasks cannot be validated by the same user who marked task as mapped or badimagery"
)
tasks_to_lock.append(task)
user_can_validate, error_reason = ProjectService.is_user_permitted_to_validate(
validation_dto.project_id, validation_dto.user_id)
if not user_can_validate:
if error_reason == ValidatingNotAllowed.USER_NOT_ACCEPTED_LICENSE:
raise UserLicenseError(
"User must accept license to map this task")
elif error_reason == ValidatingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED:
raise ValidatorServiceError("User already has a task locked")
else:
raise ValidatorServiceError(
f"Validation not allowed because: {error_reason}")
# Lock all tasks for validation
dtos = []
for task in tasks_to_lock:
task.lock_task_for_validating(validation_dto.user_id)
dtos.append(
task.as_dto_with_instructions(validation_dto.preferred_locale))
task_dtos = TaskDTOs()
task_dtos.tasks = dtos
return task_dtos
def get_task(task_id: int, project_id: int) -> Task:
"""
Get task from DB
:raises: NotFound
"""
task = Task.get(task_id, project_id)
if task is None:
raise NotFound()
return task
def as_dto_for_mapping(self,
authenticated_user_id: int = None,
locale: str = "en",
abbrev: bool = True) -> Optional[ProjectDTO]:
""" Creates a Project DTO suitable for transmitting to mapper users """
project, project_dto = self._get_project_and_base_dto()
if abbrev is False:
project_dto.tasks = Task.get_tasks_as_geojson_feature_collection(
self.id, None)
else:
project_dto.tasks = Task.get_tasks_as_geojson_feature_collection_no_geom(
self.id)
project_dto.project_info = ProjectInfo.get_dto_for_locale(
self.id, locale, project.default_locale)
if project.organisation_id:
project_dto.organisation = project.organisation.id
project_dto.organisation_name = project.organisation.name
project_dto.organisation_logo = project.organisation.logo
project_dto.project_info_locales = ProjectInfo.get_dto_for_all_locales(
self.id)
return project_dto
def is_user_permitted_to_validate(project_id, user_id):
""" Check if the user is allowed to validate on the project in scope """
if UserService.is_user_blocked(user_id):
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
project = ProjectService.get_project_by_id(project_id)
if project.license_id:
if not UserService.has_user_accepted_license(user_id, project.license_id):
return False, ValidatingNotAllowed.USER_NOT_ACCEPTED_LICENSE
validation_permission = project.validation_permission
# is_admin or is_author or is_org_manager or is_manager_team
is_manager_permission = False
if ProjectAdminService.is_user_action_permitted_on_project(user_id, project_id):
is_manager_permission = True
# Draft (public/private) accessible only for is_manager_permission
if (
ProjectStatus(project.status) == ProjectStatus.DRAFT
and not is_manager_permission
):
return False, ValidatingNotAllowed.PROJECT_NOT_PUBLISHED
is_restriction = None
if not is_manager_permission and validation_permission:
is_restriction = ProjectService.evaluate_validation_permission(
project_id, user_id, validation_permission
)
tasks = Task.get_locked_tasks_for_user(user_id)
if len(tasks.locked_tasks) > 0:
return False, ValidatingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED
is_allowed_user = None
if project.private and not is_manager_permission:
# Check if user is in allowed user list
is_allowed_user = ProjectService.is_user_in_the_allowed_list(
project.allowed_users, user_id
)
if is_allowed_user:
return True, "User allowed to validate"
if not is_manager_permission and is_restriction:
return is_restriction
elif project.private and not (
is_manager_permission or is_allowed_user or not is_restriction
):
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
return True, "User allowed to validate"
def test_per_task_instructions_returns_instructions_with_extra_properties(
self):
# Arrange
test_task = Task()
test_task.extra_properties = '{"foo": "bar"}'
test_task.x = 1
test_task.y = 2
test_task.zoom = 3
test_task.is_square = True
# Act
instructions = test_task.format_per_task_instructions(
"Foo is replaced by {foo}")
# Assert
self.assertEqual(instructions, "Foo is replaced by bar")
def is_user_permitted_to_validate(project_id, user_id):
""" Check if the user is allowed to validate on the project in scope """
if UserService.is_user_blocked(user_id):
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
project = ProjectService.get_project_by_id(project_id)
if project.license_id:
if not UserService.has_user_accepted_license(user_id, project.license_id):
return False, ValidatingNotAllowed.USER_NOT_ACCEPTED_LICENSE
validation_permission = project.validation_permission
is_manager_permission = False
# is_admin or is_author or is_org_manager or is_manager_team
if ProjectAdminService.is_user_action_permitted_on_project(user_id, project_id):
is_manager_permission = True
if (
ProjectStatus(project.status) != ProjectStatus.PUBLISHED
and not is_manager_permission
):
return False, ValidatingNotAllowed.PROJECT_NOT_PUBLISHED
tasks = Task.get_locked_tasks_for_user(user_id)
if len(tasks.locked_tasks) > 0:
return False, ValidatingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED
if project.private and not is_manager_permission:
# Check user is in allowed users
try:
next(user for user in project.allowed_users if user.id == user_id)
except StopIteration:
return False, ValidatingNotAllowed.USER_NOT_ON_ALLOWED_LIST
is_restriction = ProjectService.evaluate_validation_permission(
project_id, user_id, validation_permission
)
if is_restriction:
return is_restriction
if project.validation_permission and not is_manager_permission:
is_restriction = ProjectService.evaluate_validation_permission(
project_id, user_id, validation_permission
)
if is_restriction:
return is_restriction
return True, "User allowed to validate"
def test_per_task_instructions_with_underscores_formatted_correctly(self):
test_task = Task()
test_task.x = 1
test_task.y = 2
test_task.zoom = 3
test_task.is_square = True
# Act
instructions = test_task.format_per_task_instructions(
"Test Url is http://test.com/{x}_{y}_{z}")
# Assert
self.assertEqual(instructions, "Test Url is http://test.com/1_2_3")
def add_task_comment(task_comment: TaskCommentDTO) -> TaskDTO:
""" Adds the comment to the task history """
task = Task.get(task_comment.task_id, task_comment.project_id)
if task is None:
raise MappingServiceError(f"Task {task_comment.task_id} not found")
task.set_task_history(TaskAction.COMMENT, task_comment.user_id,
task_comment.comment)
# Parse comment to see if any users have been @'d
MessageService.send_message_after_comment(task_comment.user_id,
task_comment.comment,
task.id,
task_comment.project_id)
task.update()
return task.as_dto_with_instructions(task_comment.preferred_locale)
def delete_tasks(project_id: int, tasks_ids):
# Validate project exists.
project = Project.get(project_id)
if project is None:
raise NotFound({"project": project_id})
tasks = [{"id": i, "obj": Task.get(i, project_id)} for i in tasks_ids]
# In case a task is not found.
not_found = [t["id"] for t in tasks if t["obj"] is None]
if len(not_found) > 0:
raise NotFound({"tasks": not_found})
# Delete task one by one.
[t["obj"].delete() for t in tasks]
def test_record_auto_unlock_adds_autounlocked_action(
self,
mock_update,
mock_set_task_history,
mock_get_last_action,
mock_get_last_status,
):
mock_history = MagicMock()
mock_last_action = MagicMock()
mock_last_action.action = "LOCKED_FOR_MAPPING"
mock_get_last_action.return_value = mock_last_action
mock_get_last_status.return_value = TaskStatus.READY
mock_set_task_history.return_value = mock_history
test_task = Task()
test_task.locked_by = "testuser"
lock_duration = "02:00"
test_task.record_auto_unlock(lock_duration)
mock_set_task_history.assert_called_with(
action=TaskAction.AUTO_UNLOCKED_FOR_MAPPING, user_id="testuser")
self.assertEqual(mock_history.action_text, lock_duration)
self.assertEqual(test_task.locked_by, None)
mock_last_action.delete.assert_called()
def get_task_details_for_logged_in_user(user_id: int, preferred_locale: str):
""" if the user is working on a task in the project return it """
tasks = Task.get_locked_tasks_details_for_user(user_id)
if len(tasks) == 0:
raise NotFound()
# TODO put the task details in to a DTO
dtos = []
for task in tasks:
dtos.append(task.as_dto_with_instructions(preferred_locale))
task_dtos = TaskDTOs()
task_dtos.tasks = dtos
return task_dtos
def test_per_task_instructions_returns_instructions_when_no_dynamic_url_and_task_not_splittable(
self, ):
# Arrange
test_task = Task()
test_task.x = 1
test_task.y = 2
test_task.zoom = 3
test_task.is_square = False
# Act
instructions = test_task.format_per_task_instructions("Use map box")
# Assert
self.assertEqual(instructions, "Use map box")
def test_per_task_instructions_formatted_correctly(self):
# Arrange
test_task = Task()
test_task.x = 1
test_task.y = 2
test_task.zoom = 3
test_task.is_square = True
# Act
instructions = test_task.format_per_task_instructions(
"Test Url is http://test.com/{x}/{y}/{z}")
# Assert
self.assertEqual(instructions, "Test Url is http://test.com/1/2/3")
def get_tasks_locked_by_user(project_id: int, unlock_tasks, user_id: int):
"""
Returns tasks specified by project id and unlock_tasks list if found and locked for validation by user,
otherwise raises ValidatorServiceError, NotFound
:param project_id:
:param unlock_tasks: List of tasks to be unlocked
:param user_id:
:return: List of Tasks
:raises ValidatorServiceError
:raises NotFound
"""
tasks_to_unlock = []
# Loop supplied tasks to check they can all be unlocked
for unlock_task in unlock_tasks:
task = Task.get(unlock_task.task_id, project_id)
if task is None:
raise NotFound(f"Task {unlock_task.task_id} not found")
current_state = TaskStatus(task.task_status)
if current_state != TaskStatus.LOCKED_FOR_VALIDATION:
raise ValidatorServiceError(
f"Task {unlock_task.task_id} is not LOCKED_FOR_VALIDATION"
)
if task.locked_by != user_id:
raise ValidatorServiceError(
"Attempting to unlock a task owned by another user"
)
if hasattr(unlock_task, "status"):
# we know what status we ate going to be setting to on unlock
new_status = TaskStatus[unlock_task.status]
else:
new_status = None
tasks_to_unlock.append(
dict(
task=task,
new_state=new_status,
comment=unlock_task.comment,
issues=unlock_task.issues,
)
)
return tasks_to_unlock
def test_split_non_square_task(self, mock_task):
# Lock task for mapping
task = Task.get(2, self.test_project.id)
task.lock_task_for_mapping(self.test_user.id)
split_task_dto = SplitTaskDTO()
split_task_dto.user_id = self.test_user.id
split_task_dto.project_id = self.test_project.id
split_task_dto.task_id = 2
# Split tasks
expected = geojson.loads(
json.dumps(get_canned_json("non_square_split_results.json"))
)
result = SplitService._create_split_tasks(task.x, task.y, task.zoom, task)
self.assertEqual(str(expected), str(result))
def is_user_permitted_to_map(project_id: int, user_id: int):
""" Check if the user is allowed to map the on the project in scope """
if UserService.is_user_blocked(user_id):
return False, MappingNotAllowed.USER_NOT_ON_ALLOWED_LIST
project = ProjectService.get_project_by_id(project_id)
mapping_permission = project.mapping_permission
if ProjectStatus(
project.status
) != ProjectStatus.PUBLISHED and not ProjectAdminService.is_user_action_permitted_on_project(
user_id, project_id):
return False, MappingNotAllowed.PROJECT_NOT_PUBLISHED
tasks = Task.get_locked_tasks_for_user(user_id)
if len(tasks.locked_tasks) > 0:
return False, MappingNotAllowed.USER_ALREADY_HAS_TASK_LOCKED
if project.private:
# Check user is in allowed users
try:
next(user for user in project.allowed_users
if user.id == user_id)
except StopIteration:
return False, MappingNotAllowed.USER_NOT_ON_ALLOWED_LIST
is_restriction = ProjectService.evaluate_mapping_permission(
project_id, user_id, mapping_permission)
if is_restriction:
return is_restriction
if project.mapping_permission:
is_restriction = ProjectService.evaluate_mapping_permission(
project_id, user_id, mapping_permission)
if is_restriction:
return is_restriction
if project.license_id:
if not UserService.has_user_accepted_license(
user_id, project.license_id):
return False, MappingNotAllowed.USER_NOT_ACCEPTED_LICENSE
return True, "User allowed to map"
def generate_gpx(project_id: int, task_ids_str: str, timestamp=None):
"""
Creates a GPX file for supplied tasks. Timestamp is for unit testing only.
You can use the following URL to test locally:
http://www.openstreetmap.org/edit?editor=id&#map=11/31.50362930069913/34.628906243797054&comment=CHANGSET_COMMENT&gpx=http://localhost:5000/api/v2/projects/{project_id}/tasks/queries/gpx%3Ftasks=2
"""
if timestamp is None:
timestamp = datetime.datetime.utcnow()
root = ET.Element(
"gpx",
attrib=dict(
xmlns="http://www.topografix.com/GPX/1/1",
version="1.1",
creator="HOT Tasking Manager",
),
)
# Create GPX Metadata element
metadata = ET.Element("metadata")
link = ET.SubElement(
metadata,
"link",
attrib=dict(href="https://github.com/hotosm/tasking-manager"),
)
ET.SubElement(link, "text").text = "HOT Tasking Manager"
ET.SubElement(metadata, "time").text = timestamp.isoformat()
root.append(metadata)
# Create trk element
trk = ET.Element("trk")
root.append(trk)
ET.SubElement(
trk, "name"
).text = f"Task for project {project_id}. Do not edit outside of this area!"
# Construct trkseg elements
if task_ids_str is not None:
task_ids = map(int, task_ids_str.split(","))
tasks = Task.get_tasks(project_id, task_ids)
if not tasks or len(tasks) == 0:
raise NotFound()
else:
tasks = Task.get_all_tasks(project_id)
if not tasks or len(tasks) == 0:
raise NotFound()
for task in tasks:
task_geom = shape.to_shape(task.geometry)
for poly in task_geom:
trkseg = ET.SubElement(trk, "trkseg")
for point in poly.exterior.coords:
ET.SubElement(
trkseg,
"trkpt",
attrib=dict(lon=str(point[0]), lat=str(point[1])),
)
# Append wpt elements to end of doc
wpt = ET.Element("wpt",
attrib=dict(lon=str(point[0]),
lat=str(point[1])))
root.append(wpt)
xml_gpx = ET.tostring(root, encoding="utf8")
return xml_gpx
