def generate_gpx(project_id: int, task_ids_str: str, timestamp=None):
"""
Creates a GPX file for supplied tasks. Timestamp is for unit testing only. You can use the following URL to test locally:
http://www.openstreetmap.org/edit?editor=id&#map=11/31.50362930069913/34.628906243797054&comment=CHANGSET_COMMENT&gpx=http://localhost:5000/api/v1/project/111/tasks_as_gpx%3Ftasks=2
"""
if timestamp is None:
timestamp = datetime.datetime.utcnow()
root = ET.Element('gpx',
attrib=dict(
xmlns='http://www.topografix.com/GPX/1/1',
version='1.1',
creator='HOT Tasking Manager'))
# Create GPX Metadata element
metadata = ET.Element('metadata')
link = ET.SubElement(
metadata,
'link',
attrib=dict(href='https://github.com/hotosm/tasking-manager'))
ET.SubElement(link, 'text').text = 'HOT Tasking Manager'
ET.SubElement(metadata, 'time').text = timestamp.isoformat()
root.append(metadata)
# Create trk element
trk = ET.Element('trk')
root.append(trk)
ET.SubElement(
trk, 'name'
).text = f'Task for project {project_id}. Do not edit outside of this area!'
# Construct trkseg elements
if task_ids_str is not None:
task_ids = map(int, task_ids_str.split(','))
tasks = Task.get_tasks(project_id, task_ids)
if not tasks or len(tasks) == 0:
raise NotFound()
else:
tasks = Task.get_all_tasks(project_id)
if not tasks or len(tasks) == 0:
raise NotFound()
for task in tasks:
task_geom = shape.to_shape(task.geometry)
for poly in task_geom:
trkseg = ET.SubElement(trk, 'trkseg')
for point in poly.exterior.coords:
ET.SubElement(trkseg,
'trkpt',
attrib=dict(lon=str(point[0]),
lat=str(point[1])))
# Append wpt elements to end of doc
wpt = ET.Element('wpt',
attrib=dict(lon=str(point[0]),
lat=str(point[1])))
root.append(wpt)
xml_gpx = ET.tostring(root, encoding='utf8')
return xml_gpx
def generate_osm_xml(project_id: int, task_ids_str: str) -> str:
""" Generate xml response suitable for loading into JOSM. A sample output file is in
/server/helpers/testfiles/osm-sample.xml """
# Note XML created with upload No to ensure it will be rejected by OSM if uploaded by mistake
root = ET.Element('osm', attrib=dict(version='0.6', upload='never', creator='HOT Tasking Manager'))
if task_ids_str:
task_ids = map(int, task_ids_str.split(','))
tasks = Task.get_tasks(project_id, task_ids)
if not tasks or tasks.count() == 0:
raise NotFound()
else:
tasks = Task.get_all_tasks(project_id)
if not tasks or len(tasks) == 0:
raise NotFound()
fake_id = -1 # We use fake-ids to ensure XML will not be validated by OSM
for task in tasks:
task_geom = shape.to_shape(task.geometry)
way = ET.SubElement(root, 'way', attrib=dict(id=str((task.id * -1)), action='modify', visible='true'))
for poly in task_geom:
for point in poly.exterior.coords:
ET.SubElement(root, 'node', attrib=dict(action='modify', visible='true', id=str(fake_id),
lon=str(point[0]), lat=str(point[1])))
ET.SubElement(way, 'nd', attrib=dict(ref=str(fake_id)))
fake_id -= 1
xml_gpx = ET.tostring(root, encoding='utf8')
return xml_gpx
def get_user_contributions(project_id: int) -> ProjectContributionsDTO:
""" Get all user contributions on a project"""
contrib_query = '''select m.mapped_by, m.username, m.mapped, v.validated_by, v.username, v.validated
from (select t.mapped_by, u.username, count(t.mapped_by) mapped
from tasks t,
users u
where t.mapped_by = u.id
and t.project_id = {0}
and t.mapped_by is not null
group by t.mapped_by, u.username) m FULL OUTER JOIN
(select t.validated_by, u.username, count(t.validated_by) validated
from tasks t,
users u
where t.validated_by = u.id
and t.project_id = {0}
and t.validated_by is not null
group by t.validated_by, u.username) v
ON m.mapped_by = v.validated_by
'''.format(project_id)
results = db.engine.execute(contrib_query)
if results.rowcount == 0:
raise NotFound()
contrib_dto = ProjectContributionsDTO()
for row in results:
user_contrib = UserContribution()
user_contrib.username = row[1] if row[1] else row[4]
user_contrib.mapped = row[2] if row[2] else 0
user_contrib.validated = row[5] if row[5] else 0
contrib_dto.user_contributions.append(user_contrib)
return contrib_dto
def get_project_by_id(project_id: int) -> Project:
project = Project.get(project_id)
if project is None:
raise NotFound()
return project
def get_mapped_tasks_by_user(project_id: int):
""" Gets all mapped tasks for supplied project grouped by user"""
# Raw SQL is easier to understand that SQL alchemy here :)
sql = """select u.username, u.mapping_level, count(distinct(t.id)), json_agg(distinct(t.id)),
max(th.action_date) last_seen, u.date_registered, u.last_validation_date
from tasks t,
task_history th,
users u
where t.project_id = th.project_id
and t.id = th.task_id
and t.mapped_by = u.id
and t.project_id = {0}
and t.task_status = 2
and th.action_text = 'MAPPED'
group by u.username, u.mapping_level, u.date_registered, u.last_validation_date""".format(project_id)
results = db.engine.execute(sql)
if results.rowcount == 0:
raise NotFound()
mapped_tasks_dto = MappedTasks()
for row in results:
user_mapped = MappedTasksByUser()
user_mapped.username = row[0]
user_mapped.mapping_level = MappingLevel(row[1]).name
user_mapped.mapped_task_count = row[2]
user_mapped.tasks_mapped = row[3]
user_mapped.last_seen = row[4]
user_mapped.date_registered = row[5]
user_mapped.last_validation_date = row[6]
mapped_tasks_dto.mapped_tasks.append(user_mapped)
return mapped_tasks_dto
def filter_users(user_filter: str, project_id: int,
page: int) -> UserFilterDTO:
""" Finds users that matches first characters, for auto-complete.
Users who have participated (mapped or validated) in the project, if given, will be
returned ahead of those who have not.
"""
# Note that the projects_mapped column includes both mapped and validated projects.
results = db.session.query(User.username, User.projects_mapped.any(project_id).label("participant")) \
.filter(User.username.ilike(user_filter.lower() + '%')) \
.order_by(desc("participant").nullslast(), User.username).paginate(page, 20, True)
if results.total == 0:
raise NotFound()
dto = UserFilterDTO()
for result in results.items:
dto.usernames.append(result.username)
if project_id is not None:
participant = ProjectParticipantUser()
participant.username = result.username
participant.project_id = project_id
participant.is_participant = bool(result.participant)
dto.users.append(participant)
dto.pagination = Pagination(results)
return dto
def get_mapped_projects(user_id: int, preferred_locale: str) -> UserMappedProjectsDTO:
""" Get all projects a user has mapped on """
sql = '''select p.id, p.status, p.default_locale, count(t.mapped_by), count(t.validated_by), st_asgeojson(p.centroid),
st_asgeojson(p.geometry)
from projects p,
tasks t
where p.id in (select unnest(projects_mapped) from users where id = {0})
and p.id = t.project_id
and (t.mapped_by = {0} or t.mapped_by is null)
and (t.validated_by = {0} or t.validated_by is null)
GROUP BY p.id, p.status, p.centroid, p.geometry'''.format(user_id)
results = db.engine.execute(sql)
if results.rowcount == 0:
raise NotFound()
mapped_projects_dto = UserMappedProjectsDTO()
for row in results:
mapped_project = MappedProject()
mapped_project.project_id = row[0]
mapped_project.status = ProjectStatus(row[1]).name
mapped_project.tasks_mapped = row[3]
mapped_project.tasks_validated = row[4]
mapped_project.centroid = geojson.loads(row[5])
mapped_project.aoi = geojson.loads(row[6])
project_info = ProjectInfo.get_dto_for_locale(row[0], preferred_locale, row[2])
mapped_project.name = project_info.name
mapped_projects_dto.mapped_projects.append(mapped_project)
return mapped_projects_dto
def get_user_by_id(user_id: int) -> User:
user = User().get_by_id(user_id)
if user is None:
raise NotFound()
return user
def get_token(token: str):
application = Application.get_token(token)
if application is None:
raise NotFound()
return application
def get_projects_for_admin(admin_id: int, preferred_locale: str) -> PMDashboardDTO:
""" Get projects for admin """
admins_projects = db.session.query(Project.id,
Project.status,
Project.campaign_tag,
Project.total_tasks,
Project.tasks_mapped,
Project.tasks_validated,
Project.tasks_bad_imagery,
Project.created,
Project.last_updated,
Project.default_locale,
AreaOfInterest.centroid.ST_AsGeoJSON().label('geojson'))\
.join(AreaOfInterest).filter(Project.author_id == admin_id).all()
if admins_projects is None:
raise NotFound('No projects found for admin')
admin_projects_dto = PMDashboardDTO()
for project in admins_projects:
pm_project = Project.get_project_summary(project, preferred_locale)
project_status = ProjectStatus(project.status)
if project_status == ProjectStatus.DRAFT:
admin_projects_dto.draft_projects.append(pm_project)
elif project_status == ProjectStatus.PUBLISHED:
admin_projects_dto.active_projects.append(pm_project)
elif project_status == ProjectStatus.ARCHIVED:
admin_projects_dto.archived_projects.append(pm_project)
else:
current_app.logger.error(f'Unexpected state project {project.id}')
return admin_projects_dto
def get_latest_activity(project_id: int, page: int) -> ProjectActivityDTO:
""" Gets all the activity on a project """
results = db.session.query(
TaskHistory.id, TaskHistory.task_id, TaskHistory.action, TaskHistory.action_date,
TaskHistory.action_text, User.username
).join(User).filter(
TaskHistory.project_id == project_id,
TaskHistory.action != 'COMMENT'
).order_by(
TaskHistory.action_date.desc()
).paginate(page, 10, True)
if results.total == 0:
raise NotFound()
activity_dto = ProjectActivityDTO()
for item in results.items:
history = TaskHistoryDTO()
history.history_id = item.id
history.task_id = item.task_id
history.action = item.action
history.action_text = item.action_text
history.action_date = item.action_date
history.action_by = item.username
activity_dto.activity.append(history)
activity_dto.pagination = Pagination(results)
return activity_dto
def get_user_by_username(username: str) -> User:
user = User().get_by_username(username)
if user is None:
raise NotFound()
return user
def get_layer_by_id(layer_id: int) -> Layer:
""" Returns a layer by ID """
layer = Layer.get_by_id(layer_id)
if layer is None:
raise NotFound()
return layer
def get_all_layers(locale: str = 'en') -> DMISLayersDTO:
""" Returns a list of layers """
layers = Layer.get_all_layers(locale)
if layers is None:
raise NotFound()
return layers
def get_layer_dto_by_id(layer_id: int) -> LayerDetailsDTO:
""" Returns a layer by ID """
layer = Layer.get_by_id(layer_id)
if layer is None:
raise NotFound()
return layer.as_dto()
def get_max_task_id_for_project(project_id: int):
"""Gets the nights task id currently in use on a project"""
sql = """select max(id) from tasks where project_id = {0} GROUP BY project_id""".format(project_id)
result = db.engine.execute(sql)
if result.rowcount == 0:
raise NotFound()
for row in result:
return row[0]
def get_all_comments(project_id: int) -> ProjectCommentsDTO:
""" Gets all comments mappers, validators have added to tasks associated with project """
comments = TaskHistory.get_all_comments(project_id)
if len(comments.comments) == 0:
raise NotFound('No comments found on project')
return comments
def get_user_contributions(project_id: int) -> ProjectContributionsDTO:
""" Get all user contributions on a project"""
contrib_query = '''select m.mapped_by, m.username, m.mapped, v.validated_by, v.username, v.validated
from (select t.mapped_by, u.username, count(t.mapped_by) mapped
from tasks t,
users u
where t.mapped_by = u.id
and t.project_id = {0}
and t.mapped_by is not null
group by t.mapped_by, u.username) m FULL OUTER JOIN
(select t.validated_by, u.username, count(t.validated_by) validated
from tasks t,
users u
where t.validated_by = u.id
and t.project_id = {0}
and t.validated_by is not null
group by t.validated_by, u.username) v
ON m.mapped_by = v.validated_by
'''.format(project_id)
results = db.engine.execute(contrib_query)
if results.rowcount == 0:
raise NotFound()
contrib_dto = ProjectContributionsDTO()
for row in results:
if row[0]:
user_contrib = UserContribution()
user_contrib.username = row[1] if row[1] else row[4]
user_contrib.mapped = row[2] if row[2] else 0
user_contrib.validated = row[5] if row[5] else 0
user_contrib.total_time_spent = 0
user_contrib.time_spent_mapping = 0
user_contrib.time_spent_validating = 0
sql = """SELECT SUM(TO_TIMESTAMP(action_text, 'HH24:MI:SS')::TIME) FROM task_history
WHERE action='LOCKED_FOR_MAPPING'
and user_id = {0} and project_id = {1};""".format(row[0], project_id)
total_mapping_time = db.engine.execute(sql)
for time in total_mapping_time:
total_mapping_time = time[0]
if total_mapping_time:
user_contrib.time_spent_mapping = total_mapping_time.total_seconds()
user_contrib.total_time_spent += user_contrib.time_spent_mapping
sql = """SELECT SUM(TO_TIMESTAMP(action_text, 'HH24:MI:SS')::TIME) FROM task_history
WHERE action='LOCKED_FOR_VALIDATION'
and user_id = {0} and project_id = {1};""".format(row[0], project_id)
total_validation_time = db.engine.execute(sql)
for time in total_validation_time:
total_validation_time = time[0]
if total_validation_time:
user_contrib.time_spent_validating = total_validation_time.total_seconds()
user_contrib.total_time_spent += user_contrib.time_spent_validating
contrib_dto.user_contributions.append(user_contrib)
return contrib_dto
def get_user_by_id(user_id: int) -> User:
""" Returns user that matches ID """
# TODO cache this
user = User.get_by_id(user_id)
if user is None:
raise NotFound()
return user
def get_mapped_projects(user_id: int,
preferred_locale: str) -> UserMappedProjectsDTO:
""" Get all projects a user has mapped on """
# This query looks scary, but we're really just creating an outer join between the query that gets the
# counts of all mapped tasks and the query that gets counts of all validated tasks. This is necessary to
# handle cases where users have only validated tasks on a project, or only mapped on a project.
sql = '''SELECT p.id,
p.status,
p.default_locale,
c.mapped,
c.validated,
st_asgeojson(p.centroid)
FROM projects p,
(SELECT coalesce(v.project_id, m.project_id) project_id,
coalesce(v.validated, 0) validated,
coalesce(m.mapped, 0) mapped
FROM (SELECT t.project_id,
count (t.validated_by) validated
FROM tasks t
WHERE t.project_id IN (SELECT unnest(projects_mapped) FROM users WHERE id = {0})
AND t.validated_by = {0}
GROUP BY t.project_id, t.validated_by) v
FULL OUTER JOIN
(SELECT t.project_id,
count(t.mapped_by) mapped
FROM tasks t
WHERE t.project_id IN (SELECT unnest(projects_mapped) FROM users WHERE id = {0})
AND t.mapped_by = {0}
GROUP BY t.project_id, t.mapped_by) m
ON v.project_id = m.project_id) c
WHERE p.id = c.project_id ORDER BY p.id DESC'''.format(
user_id)
results = db.engine.execute(sql)
if results.rowcount == 0:
raise NotFound()
mapped_projects_dto = UserMappedProjectsDTO()
for row in results:
mapped_project = MappedProject()
mapped_project.project_id = row[0]
mapped_project.status = ProjectStatus(row[1]).name
mapped_project.tasks_mapped = row[3]
mapped_project.tasks_validated = row[4]
mapped_project.centroid = geojson.loads(row[5])
project_info = ProjectInfo.get_dto_for_locale(
row[0], preferred_locale, row[2])
mapped_project.name = project_info.name
mapped_projects_dto.mapped_projects.append(mapped_project)
return mapped_projects_dto
def search_projects(
search_dto: ProjectSearchDTO) -> ProjectSearchResultsDTO:
""" Searches all projects for matches to the criteria provided by the user """
all_results, paginated_results = ProjectSearchService._filter_projects(
search_dto)
if paginated_results.total == 0:
raise NotFound()
features = []
for project in all_results:
# This loop creates a geojson feature collection so you can see all active projects on the map
properties = {
"projectId": project.id,
"priority": ProjectPriority(project.priority).name
}
centroid = project.centroid
feature = geojson.Feature(geometry=geojson.loads(project.centroid),
properties=properties)
features.append(feature)
feature_collection = geojson.FeatureCollection(features)
dto = ProjectSearchResultsDTO()
dto.map_results = feature_collection
for project in paginated_results.items:
# This loop loads the paginated text results
# TODO would be nice to get this for an array rather than individually would be more efficient
project_info_dto = ProjectInfo.get_dto_for_locale(
project.id, search_dto.preferred_locale,
project.default_locale)
list_dto = ListSearchResultDTO()
list_dto.project_id = project.id
list_dto.locale = project_info_dto.locale
list_dto.name = project_info_dto.name
list_dto.priority = ProjectPriority(project.priority).name
list_dto.mapper_level = MappingLevel(project.mapper_level).name
list_dto.short_description = project_info_dto.short_description
list_dto.organisation_tag = project.organisation_tag
list_dto.campaign_tag = project.campaign_tag
list_dto.percent_mapped = Project.calculate_tasks_percent(
'mapped', project.total_tasks, project.tasks_mapped,
project.tasks_validated, project.tasks_bad_imagery)
list_dto.percent_validated = Project.calculate_tasks_percent(
'validated', project.total_tasks, project.tasks_mapped,
project.tasks_validated, project.tasks_bad_imagery)
list_dto.status = ProjectStatus(project.status).name
list_dto.active_mappers = Project.get_active_mappers(project.id)
dto.results.append(list_dto)
dto.pagination = Pagination(paginated_results)
return dto
def get_license(license_id: int) -> License:
"""
Get task from DB
:raises: NotFound
"""
map_license = License.get_by_id(license_id)
if map_license is None:
raise NotFound()
return map_license
def get_task(task_id: int, project_id: int) -> Task:
"""
Get task from DB
:raises: NotFound
"""
task = Task.get(task_id, project_id)
if task is None:
raise NotFound()
return task
def get_task_for_logged_in_user(project_id: int, user_id: int):
""" if the user is working on a task in the project return it """
project = ProjectService.get_project_by_id(project_id)
tasks = project.get_locked_tasks_for_user(user_id)
if len(tasks) == 0:
raise NotFound()
tasks_dto = LockedTasksForUser()
tasks_dto.locked_tasks = tasks
return tasks_dto
def get_all_users():
""" Return a list of all users """
user_list = User.query.all()
if len(user_list) == 0:
raise NotFound()
user_list_dto = UserListDTO()
for user in user_list:
user_list_dto.user_list.append(user.as_dto())
return user_list_dto
def get_all_messages(user_id: int) -> MessagesDTO:
""" Gets all messages to the user """
user_messages = Message.query.filter(Message.to_user_id == user_id).all()
if len(user_messages) == 0:
raise NotFound()
messages_dto = MessagesDTO()
for message in user_messages:
messages_dto.user_messages.append(message.as_dto())
return messages_dto
def lock_tasks_for_validation(
validation_dto: LockForValidationDTO) -> TaskDTOs:
"""
Lock supplied tasks for validation
:raises ValidatatorServiceError
"""
# Loop supplied tasks to check they can all be locked for validation
tasks_to_lock = []
for task_id in validation_dto.task_ids:
task = Task.get(task_id, validation_dto.project_id)
if task is None:
raise NotFound(f'Task {task_id} not found')
if TaskStatus(task.task_status) not in [
TaskStatus.MAPPED, TaskStatus.VALIDATED,
TaskStatus.BADIMAGERY
]:
raise ValidatatorServiceError(
f'Task {task_id} is not MAPPED, BADIMAGERY or VALIDATED')
if not ValidatorService._user_can_validate_task(
validation_dto.user_id, task.mapped_by):
raise ValidatatorServiceError(
f'Tasks cannot be validated by the same user who marked task as mapped or badimagery'
)
tasks_to_lock.append(task)
user_can_validate, error_reason = ProjectService.is_user_permitted_to_validate(
validation_dto.project_id, validation_dto.user_id)
if not user_can_validate:
if error_reason == ValidatingNotAllowed.USER_NOT_ACCEPTED_LICENSE:
raise UserLicenseError(
'User must accept license to map this task')
else:
raise ValidatatorServiceError(
f'Mapping not allowed because: {error_reason.name}')
# Lock all tasks for validation
dtos = []
for task in tasks_to_lock:
task.lock_task_for_validating(validation_dto.user_id)
dtos.append(
task.as_dto_with_instructions(validation_dto.preferred_locale))
task_dtos = TaskDTOs()
task_dtos.tasks = dtos
return task_dtos
def filter_users(user_filter: str, page: int) -> UserFilterDTO:
""" Finds users that matches first characters, for auto-complete """
results = db.session.query(User.username).filter(User.username.ilike(user_filter.lower() + '%')) \
.order_by(User.username).paginate(page, 20, True)
if results.total == 0:
raise NotFound()
dto = UserFilterDTO()
for result in results.items:
dto.usernames.append(result.username)
dto.pagination = Pagination(results)
return dto
def get_all() -> LicenseListDTO:
""" Gets all licenses currently stored """
results = License.query.all()
if len(results) == 0:
raise NotFound()
dto = LicenseListDTO()
for result in results:
imagery_license = LicenseDTO()
imagery_license.license_id = result.id
imagery_license.name = result.name
imagery_license.description = result.description
imagery_license.plain_text = result.plain_text
dto.licenses.append(imagery_license)
return dto
def get_task_details_for_logged_in_user(project_id: int, user_id: int, preferred_locale: str):
""" if the user is working on a task in the project return it """
project = ProjectService.get_project_by_id(project_id)
tasks = project.get_locked_tasks_details_for_user(user_id)
if len(tasks) == 0:
raise NotFound()
# TODO put the task details in to a DTO
dtos = []
for task in tasks:
dtos.append(task.as_dto_with_instructions(preferred_locale))
task_dtos = TaskDTOs()
task_dtos.tasks = dtos
return task_dtos
