def create_directories() -> None:
"""Create directories for statistics"""
dirs = (
DATA_PATH + "/api",
DATA_PATH + "/api/agg_results",
DATA_PATH + "/api/groups",
DATA_PATH + "/api/history",
DATA_PATH + "/api/hot_tm",
DATA_PATH + "/api/projects",
DATA_PATH + "/api/results",
DATA_PATH + "/api/tasks",
DATA_PATH + "/api/yes_maybe",
DATA_PATH + "/api/users",
DATA_PATH + "/api/project_geometries",
)
for path in dirs:
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError:
logger.exception(
"Creation of the directory {0} failed".format(path))
sentry.capture_exception()
break
else:
print("Successfully created the directory {0}".format(path))
def run_create_tutorials() -> None:
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/tutorialDrafts/")
tutorial_drafts = ref.get()
if tutorial_drafts is None:
logger.info("There are no tutorial drafts in firebase.")
return None
for tutorial_draft_id, tutorial_draft in tutorial_drafts.items():
tutorial_draft["tutorialDraftId"] = tutorial_draft_id
project_type = tutorial_draft["projectType"]
project_name = tutorial_draft["name"]
try:
tutorial = ProjectType(project_type).tutorial(tutorial_draft)
tutorial.create_tutorial_groups()
tutorial.create_tutorial_tasks()
tutorial.save_tutorial()
send_slack_message(MessageType.SUCCESS, project_name,
tutorial.projectId)
logger.info(f"Success: Tutorial Creation ({project_name})")
except CustomError:
ref = fb_db.reference(f"v2/tutorialDrafts/{tutorial_draft_id}")
ref.set({})
send_slack_message(MessageType.FAIL, project_name,
tutorial.projectId)
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def update_groups_table(project_id: str):
"""Remove duplicates in 'project_types_specifics' attribute in groups table."""
logger.info(f"Start process for project: '{project_id}'")
p_con = auth.postgresDB()
query = """
UPDATE groups
SET project_type_specifics = project_type_specifics::jsonb
#- '{projectId}'
#- '{id}'
#- '{requiredCount}'
#- '{finishedCount}'
#- '{neededCount}'
#- '{reportCount}'
#- '{distributedCount}'
WHERE project_id = %(project_id)s
"""
try:
p_con.query(query, {"project_id": project_id})
logger.info(f"Updated tasks table for project '{project_id}'.")
except Exception as e:
sentry.capture_exception(e)
sentry.capture_message(
f"Could NOT update tasks table for project '{project_id}'.")
logger.exception(e)
logger.warning(
f"Could NOT update tasks table for project '{project_id}'.")
def run_user_management(email, emails, action, team_id) -> None:
"""
Manage project manager credentials
Grant or remove credentials.
"""
if email:
email_list = [email]
else:
email_list = emails
for email in email_list:
try:
if action == "add-manager-rights":
user_management.set_project_manager_rights(email)
elif action == "remove-manager-rights":
user_management.remove_project_manager_rights(email)
elif action == "add-team":
if not team_id:
click.echo("Missing argument: --team_id")
return None
user_management.add_user_to_team(email, team_id)
elif action == "remove-team":
user_management.remove_user_from_team(email)
except Exception:
logger.exception("grant user credentials failed")
sentry.capture_exception()
def run_create_projects():
"""
Create projects from submitted project drafts.
Get project drafts from Firebase.
Create projects with groups and tasks.
Save created projects, groups and tasks to Firebase and Postgres.
"""
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/projectDrafts/")
project_drafts = ref.get()
if project_drafts is None:
logger.info("There are no project drafts in firebase.")
return None
for project_draft_id, project_draft in project_drafts.items():
project_draft["projectDraftId"] = project_draft_id
project_type = project_draft["projectType"]
project_name = project_draft["name"]
try:
# Create a project object using appropriate class (project type).
project = ProjectType(project_type).constructor(project_draft)
# TODO: here the project.geometry attribute is overwritten
# this is super confusing since it's not a geojson anymore
# but this is what we set initially,
# e.g. in tile_map_service_grid/project.py
# project.geometry is set to a list of wkt geometries now
# this can't be handled in postgres,
# postgres expects just a string not an array
# validated_geometries should be called during init already
# for the respective project types
project.geometry = project.validate_geometries()
project.create_groups()
project.calc_required_results()
# Save project and its groups and tasks to Firebase and Postgres.
project.save_project()
send_slack_message(MessageType.SUCCESS, project_name,
project.projectId)
logger.info("Success: Project Creation ({0})".format(project_name))
except CustomError as e:
ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}")
ref.set({})
# check if project could be initialized
try:
project_id = project.projectId
except UnboundLocalError:
project_id = None
send_slack_message(MessageType.FAIL, project_name, project_id,
str(e))
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def run_team_management(team_name, team_id, action) -> None:
"""Create, Delete Teams or Renew TeamToken."""
try:
if action == "create":
if not team_name:
click.echo("Missing argument: --team_name")
return None
else:
team_management.create_team(team_name)
elif action == "delete":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
click.echo(
f"Do you want to delete the team with the id: {team_id}? [y/n] ",
nl=False,
)
click.echo()
c = click.getchar()
if c == "y":
click.echo("Start deletion")
team_management.delete_team(team_id)
elif c == "n":
click.echo("Abort!")
else:
click.echo("Invalid input")
elif action == "renew-team-token":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
team_management.renew_team_token(team_id)
elif action == "remove-all-team-members":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
click.echo(
f"Do you want to remove all users from "
f"the team with the id: {team_id}? [y/n] ",
nl=False,
)
click.echo()
c = click.getchar()
if c == "y":
click.echo("Start remove all team members")
team_management.remove_all_team_members(team_id)
elif c == "n":
click.echo("Abort!")
else:
click.echo("Invalid input")
except Exception:
logger.exception("team management failed")
sentry.capture_exception()
def get_last_updated_timestamp() -> str:
"""Get the timestamp of the latest created user in Postgres."""
pg_db = auth.postgresDB()
query = """
SELECT created
FROM users
WHERE created IS NOT NULL
ORDER BY created DESC
LIMIT 1
"""
last_updated = pg_db.retr_query(query)
try:
last_updated = last_updated[0][0]
last_updated = last_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
logger.info("Last updated users: {0}".format(last_updated))
except (IndexError, AttributeError):
logger.exception("Could not get last timestamp of users.")
sentry.capture_exception()
last_updated = ""
return last_updated
def results_to_file(results, projectId):
"""
Writes results to an in-memory file like object
formatted as a csv using the buffer module (StringIO).
This can be then used by the COPY statement of Postgres
for a more efficient import of many results into the Postgres
instance.
Parameters
----------
results: dict
The results as retrived from the Firebase Realtime Database instance.
Returns
-------
results_file: io.StingIO
The results in an StringIO buffer.
"""
# If csv file is a file object, it should be opened with newline=''
results_file = io.StringIO("")
w = csv.writer(results_file, delimiter="\t", quotechar="'")
logger.info(f"Got %s groups for project {projectId} to transfer" %
len(results.items()))
for groupId, users in results.items():
for userId, results in users.items():
# check if all attributes are set,
# if not don't transfer the results for this group
try:
start_time = results["startTime"]
end_time = results["endTime"]
results = results["results"]
except KeyError as e:
sentry.capture_exception(e)
sentry.capture_message(
f"at least one missing attribute for: "
f"{projectId}/{groupId}/{userId}, will skip this one")
logger.exception(e)
logger.warning(
f"at least one missing attribute for: "
f"{projectId}/{groupId}/{userId}, will skip this one")
continue
start_time = dateutil.parser.parse(start_time)
end_time = dateutil.parser.parse(end_time)
timestamp = end_time
if type(results) is dict:
for taskId, result in results.items():
w.writerow([
projectId,
groupId,
userId,
taskId,
timestamp,
start_time,
end_time,
result,
])
elif type(results) is list:
# TODO: optimize for performance
# (make sure data from firebase is always a dict)
# if key is a integer firebase will return a list
# if first key (list index) is 5
# list indicies 0-4 will have value None
for taskId, result in enumerate(results):
if result is None:
continue
else:
w.writerow([
projectId,
groupId,
userId,
taskId,
timestamp,
start_time,
end_time,
result,
])
else:
raise TypeError
results_file.seek(0)
return results_file
def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict:
"""
The function calculates all project related statistics.
Always save results to csv file.
Save groups and tasks to csv file, if not downloaded before.
If there are new results, this function will:
- Save aggregated results to csv file and GeoJSON file.
- Save project history to csv file.
- return the most recent statistics as a dictionary
The returned dictionary will be used by generate_stats.py
to update the projects_dynamic.csv
"""
# set filenames
results_filename = f"{DATA_PATH}/api/results/results_{project_id}.csv"
tasks_filename = f"{DATA_PATH}/api/tasks/tasks_{project_id}.csv"
groups_filename = f"{DATA_PATH}/api/groups/groups_{project_id}.csv"
agg_results_filename = f"{DATA_PATH}/api/agg_results/agg_results_{project_id}.csv"
agg_results_by_user_id_filename = f"{DATA_PATH}/api/users/users_{project_id}.csv"
project_stats_by_date_filename = f"{DATA_PATH}/api/history/history_{project_id}.csv"
# load data from postgres or local storage if already downloaded
results_df = get_results(results_filename, project_id)
if results_df is None:
logger.info(f"no results: skipping per project stats for {project_id}")
return {}
else:
groups_df = get_groups(groups_filename, project_id)
tasks_df = get_tasks(tasks_filename, project_id)
# aggregate results by task id
agg_results_df = get_agg_results_by_task_id(results_df, tasks_df)
agg_results_df.to_csv(agg_results_filename, index_label="idx")
logger.info(f"saved agg results for {project_id}: {agg_results_filename}")
geojson_functions.csv_to_geojson(agg_results_filename, "geom")
# aggregate results by user id
# TODO: solve memory issue for agg results by user id
try:
agg_results_by_user_id_df = user_stats.get_agg_results_by_user_id(
results_df, agg_results_df
)
agg_results_by_user_id_df.to_csv(
agg_results_by_user_id_filename, index_label="idx"
)
logger.info(
f"saved agg results for {project_id}: {agg_results_by_user_id_filename}"
)
except MemoryError:
sentry.capture_exception()
logger.info(f"failed to agg results by user id for {project_id}")
if any("maxar" in s for s in project_info["tile_server_names"]):
add_metadata_to_csv(agg_results_filename)
geojson_functions.add_metadata_to_geojson(agg_results_filename)
project_stats_by_date_df = project_stats_by_date.get_project_history(
results_df, groups_df
)
project_stats_by_date_df["project_id"] = project_id
project_stats_by_date_df.to_csv(project_stats_by_date_filename)
logger.info(
f"saved project stats by date for {project_id}: "
f"{project_stats_by_date_filename}"
)
# generate geometries for HOT Tasking Manager
if project_info.iloc[0]["project_type"] in [ProjectType.FOOTPRINT.value]:
# do not do this for ArbitraryGeometry / BuildingFootprint projects
logger.info(f"do NOT generate tasking manager geometries for {project_id}")
else:
tasking_manager_geometries.generate_tasking_manager_geometries(project_id)
# prepare output of function
project_stats_dict = {
"project_id": project_id,
"progress": project_stats_by_date_df["cum_progress"].iloc[-1],
"number_of_users": project_stats_by_date_df["cum_number_of_users"].iloc[-1],
"number_of_results": project_stats_by_date_df["cum_number_of_results"].iloc[
-1
],
"number_of_results_progress": project_stats_by_date_df[
"cum_number_of_results_progress"
].iloc[-1],
"day": project_stats_by_date_df.index[-1],
}
return project_stats_dict
def create_groups_txt_file(self, groups):
"""
Creates a text file containing groups information
for a specific project.
The text file is temporary and used only by BaseImport module.
Parameters
----------
projectId : int
The id of the project
groups : dict
The dictionary with the group information
Returns
-------
string
Filename
"""
if not os.path.isdir("{}/tmp".format(DATA_PATH)):
os.mkdir("{}/tmp".format(DATA_PATH))
# create txt file with header for later
# import with copy function into postgres
groups_txt_filename = f"{DATA_PATH}/tmp/raw_groups_{self.projectId}.txt"
groups_txt_file = open(groups_txt_filename, "w", newline="")
fieldnames = (
"project_id",
"group_id",
"number_of_tasks",
"finished_count",
"required_count",
"progress",
"project_type_specifics",
)
w = csv.DictWriter(
groups_txt_file,
fieldnames=fieldnames,
delimiter="\t",
quotechar="'",
)
for groupId, group in groups.items():
try:
output_dict = {
"project_id": self.projectId,
"group_id": groupId,
"number_of_tasks": group["numberOfTasks"],
"finished_count": group["finishedCount"],
"required_count": group["requiredCount"],
"progress": group["progress"],
"project_type_specifics": dict(),
}
# these common attributes don't need to be written
# to the project_type_specifics since they are
# already stored in separate columns
common_attributes = [
"projectId",
"groupId",
"numberOfTasks",
"requiredCount",
"finishedCount" "progress",
]
for key in group.keys():
if key not in common_attributes:
output_dict["project_type_specifics"][key] = group[key]
output_dict["project_type_specifics"] = json.dumps(
output_dict["project_type_specifics"]
)
w.writerow(output_dict)
except Exception as e:
logger.exception(
f"{self.projectId}"
f" - set_groups_postgres - "
f"groups missed critical information: {e}"
)
sentry.capture_exception()
groups_txt_file.close()
return groups_txt_filename
def transfer_results_for_project(project_id, results, filter_mode: bool = False):
"""Transfer the results for a specific project.
Save results into an in-memory file.
Copy the results to postgres.
Delete results in firebase.
We are NOT using a Firebase transaction functions here anymore.
This has caused problems, in situations where a lot of mappers are
uploading results to Firebase at the same time. Basically, this is
due to the behaviour of Firebase Transaction function:
"If another client writes to this location
before the new value is successfully saved,
the update function is called again with the new current value,
and the write will be retried."
(source: https://firebase.google.com/docs/reference/admin/python/firebase_admin.db#firebase_admin.db.Reference.transaction) # noqa
Using Firebase transaction on the group level
has turned out to be too slow when using "normal" queries,
e.g. without using threading. Threading should be avoided here
as well to not run into unforeseen errors.
For more details see issue #478.
"""
if results is None:
logger.info(f"{project_id}: No results in Firebase")
else:
# First we check for new users in Firebase.
# The user_id is used as a key in the postgres database for the results
# and thus users need to be inserted before results get inserted.
results_user_id_list = get_user_ids_from_results(results)
update_data.update_user_data(results_user_id_list)
try:
# Results are dumped into an in-memory file.
# This allows us to use the COPY statement to insert many
# results at relatively high speed.
results_file = results_to_file(results, project_id)
truncate_temp_results()
save_results_to_postgres(results_file, project_id, filter_mode=filter_mode)
except psycopg2.errors.ForeignKeyViolation as e:
sentry.capture_exception(e)
sentry.capture_message(
"could not transfer results to postgres due to ForeignKeyViolation: "
f"{project_id}; filter_mode={filter_mode}"
)
logger.exception(e)
logger.warning(
"could not transfer results to postgres due to ForeignKeyViolation: "
f"{project_id}; filter_mode={filter_mode}"
)
# There is an exception where additional invalid tasks are in a group.
# If that happens we arrive here and add the flag filtermode=true
# to this function, which could solve the issue in save_results_to_postgres.
# If it does not solve the issue we arrive again but
# since filtermode is already true, we will not try to transfer results again.
if not filter_mode:
transfer_results_for_project(project_id, results, filter_mode=True)
except Exception as e:
sentry.capture_exception(e)
sentry.capture_message(f"could not transfer results to postgres: {project_id}")
logger.exception(e)
logger.warning(f"could not transfer results to postgres: {project_id}")
else:
# It is important here that we first insert results into postgres
# and then delete these results from Firebase.
# In case something goes wrong during the insert, results in Firebase
# will not get deleted.
delete_results_from_firebase(project_id, results)
logger.info(f"{project_id}: Transferred results to postgres")
