def move_project_data_to_v2(project_id):
"""
Copy project information from old path to v2/projects in Firebase.
Add status=archived attribute.
Use Firebase transaction function for this.
"""
# Firebase transaction function
def transfer(current_data):
# we need to add these attributes
# since they are expected for version 2
current_data["status"] = "archived"
current_data["projectType"] = 1
current_data["projectId"] = str(project_id)
current_data["progress"] = current_data.get("progress", 0)
current_data["name"] = current_data.get("name", "unknown")
fb_db.reference("v2/projects/{0}".format(project_id)).set(current_data)
return dict()
fb_db = auth.firebaseDB()
projects_ref = fb_db.reference(f"projects/{project_id}")
try:
projects_ref.transaction(transfer)
logger.info(
f"{project_id}: Transfered project to v2 and delete in old path")
return True
except fb_db.TransactionAbortedError:
logger.exception(f"{project_id}: Firebase transaction"
f"for transferring project failed to commit")
return False
def run_create_tutorials() -> None:
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/tutorialDrafts/")
tutorial_drafts = ref.get()
if tutorial_drafts is None:
logger.info("There are no tutorial drafts in firebase.")
return None
for tutorial_draft_id, tutorial_draft in tutorial_drafts.items():
tutorial_draft["tutorialDraftId"] = tutorial_draft_id
project_type = tutorial_draft["projectType"]
project_name = tutorial_draft["name"]
try:
tutorial = ProjectType(project_type).tutorial(tutorial_draft)
tutorial.create_tutorial_groups()
tutorial.create_tutorial_tasks()
tutorial.save_tutorial()
send_slack_message(MessageType.SUCCESS, project_name,
tutorial.projectId)
logger.info(f"Success: Tutorial Creation ({project_name})")
except CustomError:
ref = fb_db.reference(f"v2/tutorialDrafts/{tutorial_draft_id}")
ref.set({})
send_slack_message(MessageType.FAIL, project_name,
tutorial.projectId)
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def create_directories() -> None:
"""Create directories for statistics"""
dirs = (
DATA_PATH + "/api",
DATA_PATH + "/api/agg_results",
DATA_PATH + "/api/groups",
DATA_PATH + "/api/history",
DATA_PATH + "/api/hot_tm",
DATA_PATH + "/api/projects",
DATA_PATH + "/api/results",
DATA_PATH + "/api/tasks",
DATA_PATH + "/api/yes_maybe",
DATA_PATH + "/api/users",
DATA_PATH + "/api/project_geometries",
)
for path in dirs:
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError:
logger.exception(
"Creation of the directory {0} failed".format(path))
sentry.capture_exception()
break
else:
print("Successfully created the directory {0}".format(path))
def update_groups_table(project_id: str):
"""Remove duplicates in 'project_types_specifics' attribute in groups table."""
logger.info(f"Start process for project: '{project_id}'")
p_con = auth.postgresDB()
query = """
UPDATE groups
SET project_type_specifics = project_type_specifics::jsonb
#- '{projectId}'
#- '{id}'
#- '{requiredCount}'
#- '{finishedCount}'
#- '{neededCount}'
#- '{reportCount}'
#- '{distributedCount}'
WHERE project_id = %(project_id)s
"""
try:
p_con.query(query, {"project_id": project_id})
logger.info(f"Updated tasks table for project '{project_id}'.")
except Exception as e:
sentry.capture_exception(e)
sentry.capture_message(
f"Could NOT update tasks table for project '{project_id}'.")
logger.exception(e)
logger.warning(
f"Could NOT update tasks table for project '{project_id}'.")
def run_user_management(email, emails, action, team_id) -> None:
"""
Manage project manager credentials
Grant or remove credentials.
"""
if email:
email_list = [email]
else:
email_list = emails
for email in email_list:
try:
if action == "add-manager-rights":
user_management.set_project_manager_rights(email)
elif action == "remove-manager-rights":
user_management.remove_project_manager_rights(email)
elif action == "add-team":
if not team_id:
click.echo("Missing argument: --team_id")
return None
user_management.add_user_to_team(email, team_id)
elif action == "remove-team":
user_management.remove_user_from_team(email)
except Exception:
logger.exception("grant user credentials failed")
sentry.capture_exception()
def transfer_results(project_id_list=None):
"""
Download results from firebase,
saves them to postgres and then deletes the results in firebase.
This is implemented as a transactional operation as described in
the Firebase docs to avoid missing new generated results in
Firebase during execution of this function.
"""
# Firebase transaction function
def transfer(current_results):
if current_results is None:
logger.info(f"{project_id}: No results in Firebase")
return dict()
else:
results_user_id_list = get_user_ids_from_results(current_results)
update_data.update_user_data(results_user_id_list)
results_file = results_to_file(current_results, project_id)
save_results_to_postgres(results_file)
return dict()
fb_db = auth.firebaseDB()
if not project_id_list:
# get project_ids from existing results if no project ids specified
project_id_list = fb_db.reference("v2/results/").get(shallow=True)
if not project_id_list:
project_id_list = []
logger.info(f"There are no results to transfer.")
# get all project ids from postgres,
# we will only transfer results for projects we have there
postgres_project_ids = get_projects_from_postgres()
for project_id in project_id_list:
if project_id not in postgres_project_ids:
logger.info(f"{project_id}: This project is not in postgres. "
f"We will not transfer results")
continue
elif "tutorial" in project_id:
logger.info(f"{project_id}: these are results for a tutorial. "
f"We will not transfer these")
continue
logger.info(f"{project_id}: Start transfering results")
results_ref = fb_db.reference(f"v2/results/{project_id}")
truncate_temp_results()
try:
results_ref.transaction(transfer)
logger.info(f"{project_id}: Transfered results to postgres")
except fb_db.TransactionAbortedError:
logger.exception(f"{project_id}: Firebase transaction for "
f"transfering results failed to commit")
del fb_db
return project_id_list
def run_create_projects():
"""
Create projects from submitted project drafts.
Get project drafts from Firebase.
Create projects with groups and tasks.
Save created projects, groups and tasks to Firebase and Postgres.
"""
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/projectDrafts/")
project_drafts = ref.get()
if project_drafts is None:
logger.info("There are no project drafts in firebase.")
return None
for project_draft_id, project_draft in project_drafts.items():
project_draft["projectDraftId"] = project_draft_id
project_type = project_draft["projectType"]
project_name = project_draft["name"]
try:
# Create a project object using appropriate class (project type).
project = ProjectType(project_type).constructor(project_draft)
# TODO: here the project.geometry attribute is overwritten
# this is super confusing since it's not a geojson anymore
# but this is what we set initially,
# e.g. in tile_map_service_grid/project.py
# project.geometry is set to a list of wkt geometries now
# this can't be handled in postgres,
# postgres expects just a string not an array
# validated_geometries should be called during init already
# for the respective project types
project.geometry = project.validate_geometries()
project.create_groups()
project.calc_required_results()
# Save project and its groups and tasks to Firebase and Postgres.
project.save_project()
send_slack_message(MessageType.SUCCESS, project_name,
project.projectId)
logger.info("Success: Project Creation ({0})".format(project_name))
except CustomError as e:
ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}")
ref.set({})
# check if project could be initialized
try:
project_id = project.projectId
except UnboundLocalError:
project_id = None
send_slack_message(MessageType.FAIL, project_name, project_id,
str(e))
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def run_team_management(team_name, team_id, action) -> None:
"""Create, Delete Teams or Renew TeamToken."""
try:
if action == "create":
if not team_name:
click.echo("Missing argument: --team_name")
return None
else:
team_management.create_team(team_name)
elif action == "delete":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
click.echo(
f"Do you want to delete the team with the id: {team_id}? [y/n] ",
nl=False,
)
click.echo()
c = click.getchar()
if c == "y":
click.echo("Start deletion")
team_management.delete_team(team_id)
elif c == "n":
click.echo("Abort!")
else:
click.echo("Invalid input")
elif action == "renew-team-token":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
team_management.renew_team_token(team_id)
elif action == "remove-all-team-members":
if not team_id:
click.echo("Missing argument: --team_id")
return None
else:
click.echo(
f"Do you want to remove all users from "
f"the team with the id: {team_id}? [y/n] ",
nl=False,
)
click.echo()
c = click.getchar()
if c == "y":
click.echo("Start remove all team members")
team_management.remove_all_team_members(team_id)
elif c == "n":
click.echo("Abort!")
else:
click.echo("Invalid input")
except Exception:
logger.exception("team management failed")
sentry.capture_exception()
def get_last_updated_timestamp() -> str:
"""Get the timestamp of the latest created user in Postgres."""
pg_db = auth.postgresDB()
query = """
SELECT created
FROM users
WHERE created IS NOT NULL
ORDER BY created DESC
LIMIT 1
"""
last_updated = pg_db.retr_query(query)
try:
last_updated = last_updated[0][0]
last_updated = last_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
logger.info("Last updated users: {0}".format(last_updated))
except (IndexError, AttributeError):
logger.exception("Could not get last timestamp of users.")
sentry.capture_exception()
last_updated = ""
return last_updated
def type_cast_value(self, ctx, value):
try:
return ast.literal_eval(value)
except ValueError as e:
logger.exception(e)
raise click.BadParameter(value)
def results_to_file(results, projectId):
"""
Writes results to an in-memory file like object
formatted as a csv using the buffer module (StringIO).
This can be then used by the COPY statement of Postgres
for a more efficient import of many results into the Postgres
instance.
Parameters
----------
results: dict
The results as retrived from the Firebase Realtime Database instance.
Returns
-------
results_file: io.StingIO
The results in an StringIO buffer.
"""
# If csv file is a file object, it should be opened with newline=''
results_file = io.StringIO("")
w = csv.writer(results_file, delimiter="\t", quotechar="'")
logger.info(f"Got %s groups for project {projectId} to transfer" %
len(results.items()))
for groupId, users in results.items():
for userId, results in users.items():
# check if all attributes are set,
# if not don't transfer the results for this group
try:
start_time = results["startTime"]
end_time = results["endTime"]
results = results["results"]
except KeyError as e:
sentry.capture_exception(e)
sentry.capture_message(
f"at least one missing attribute for: "
f"{projectId}/{groupId}/{userId}, will skip this one")
logger.exception(e)
logger.warning(
f"at least one missing attribute for: "
f"{projectId}/{groupId}/{userId}, will skip this one")
continue
start_time = dateutil.parser.parse(start_time)
end_time = dateutil.parser.parse(end_time)
timestamp = end_time
if type(results) is dict:
for taskId, result in results.items():
w.writerow([
projectId,
groupId,
userId,
taskId,
timestamp,
start_time,
end_time,
result,
])
elif type(results) is list:
# TODO: optimize for performance
# (make sure data from firebase is always a dict)
# if key is a integer firebase will return a list
# if first key (list index) is 5
# list indicies 0-4 will have value None
for taskId, result in enumerate(results):
if result is None:
continue
else:
w.writerow([
projectId,
groupId,
userId,
taskId,
timestamp,
start_time,
end_time,
result,
])
else:
raise TypeError
results_file.seek(0)
return results_file
def save_project(self):
"""
Creates a projects with groups and tasks
and saves it in firebase and postgres
Returns
------
Boolean: True = Successful
"""
logger.info(f"{self.projectId}" f" - start creating a project")
# Convert object attributes to dictionaries
# for saving it to firebase and postgres
project = vars(self)
groups = dict()
groupsOfTasks = dict()
for group in self.groups:
group = vars(group)
tasks = list()
for task in group["tasks"]:
tasks.append(vars(task))
groupsOfTasks[group["groupId"]] = tasks
del group["tasks"]
groups[group["groupId"]] = group
del project["groups"]
project.pop("inputGeometries", None)
project.pop("validInputGeometries", None)
# Convert Date object to ISO Datetime:
# https://www.w3.org/TR/NOTE-datetime
project["created"] = self.created.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
# logger.info(
# f'{self.projectId}'
# f' - size of all tasks: '
# f'{sys.getsizeof(json.dumps(groupsOfTasks))/1024/1024} MB'
# )
# Make sure projects get saved in Postgres and Firebase successful
try:
self.save_to_postgres(
project,
groups,
groupsOfTasks,
)
logger.info(
f"{self.projectId}" f" - the project has been saved" f" to postgres"
)
except Exception as e:
logger.exception(
f"{self.projectId}"
f" - the project could not be saved"
f" to postgres and will therefor not be "
f" saved to firebase"
)
raise CustomError(e)
# if project can't be saved to files, delete also in postgres
try:
self.save_to_files(project)
logger.info(
f"{self.projectId}" f" - the project has been saved" f" to files"
)
except Exception as e:
self.delete_from_postgres()
logger.exception(
f"{self.projectId}" f" - the project could not be saved" f" to files. "
)
logger.info(
f"{self.projectId} deleted project data from files and postgres"
)
raise CustomError(e)
try:
self.save_to_firebase(
project,
groups,
groupsOfTasks,
)
logger.info(
f"{self.projectId}" f" - the project has been saved" f" to firebase"
)
# if project can't be saved to firebase, delete also in postgres
except Exception as e:
self.delete_from_postgres()
self.delete_from_files()
logger.exception(
f"{self.projectId}"
f" - the project could not be saved"
f" to firebase. "
)
logger.info(
f"{self.projectId} deleted project data from postgres and files"
)
raise CustomError(e)
return True
def create_groups_txt_file(self, groups):
"""
Creates a text file containing groups information
for a specific project.
The text file is temporary and used only by BaseImport module.
Parameters
----------
projectId : int
The id of the project
groups : dict
The dictionary with the group information
Returns
-------
string
Filename
"""
if not os.path.isdir("{}/tmp".format(DATA_PATH)):
os.mkdir("{}/tmp".format(DATA_PATH))
# create txt file with header for later
# import with copy function into postgres
groups_txt_filename = f"{DATA_PATH}/tmp/raw_groups_{self.projectId}.txt"
groups_txt_file = open(groups_txt_filename, "w", newline="")
fieldnames = (
"project_id",
"group_id",
"number_of_tasks",
"finished_count",
"required_count",
"progress",
"project_type_specifics",
)
w = csv.DictWriter(
groups_txt_file,
fieldnames=fieldnames,
delimiter="\t",
quotechar="'",
)
for groupId, group in groups.items():
try:
output_dict = {
"project_id": self.projectId,
"group_id": groupId,
"number_of_tasks": group["numberOfTasks"],
"finished_count": group["finishedCount"],
"required_count": group["requiredCount"],
"progress": group["progress"],
"project_type_specifics": dict(),
}
# these common attributes don't need to be written
# to the project_type_specifics since they are
# already stored in separate columns
common_attributes = [
"projectId",
"groupId",
"numberOfTasks",
"requiredCount",
"finishedCount" "progress",
]
for key in group.keys():
if key not in common_attributes:
output_dict["project_type_specifics"][key] = group[key]
output_dict["project_type_specifics"] = json.dumps(
output_dict["project_type_specifics"]
)
w.writerow(output_dict)
except Exception as e:
logger.exception(
f"{self.projectId}"
f" - set_groups_postgres - "
f"groups missed critical information: {e}"
)
sentry.capture_exception()
groups_txt_file.close()
return groups_txt_filename
def transfer_results_for_project(project_id, results, filter_mode: bool = False):
"""Transfer the results for a specific project.
Save results into an in-memory file.
Copy the results to postgres.
Delete results in firebase.
We are NOT using a Firebase transaction functions here anymore.
This has caused problems, in situations where a lot of mappers are
uploading results to Firebase at the same time. Basically, this is
due to the behaviour of Firebase Transaction function:
"If another client writes to this location
before the new value is successfully saved,
the update function is called again with the new current value,
and the write will be retried."
(source: https://firebase.google.com/docs/reference/admin/python/firebase_admin.db#firebase_admin.db.Reference.transaction) # noqa
Using Firebase transaction on the group level
has turned out to be too slow when using "normal" queries,
e.g. without using threading. Threading should be avoided here
as well to not run into unforeseen errors.
For more details see issue #478.
"""
if results is None:
logger.info(f"{project_id}: No results in Firebase")
else:
# First we check for new users in Firebase.
# The user_id is used as a key in the postgres database for the results
# and thus users need to be inserted before results get inserted.
results_user_id_list = get_user_ids_from_results(results)
update_data.update_user_data(results_user_id_list)
try:
# Results are dumped into an in-memory file.
# This allows us to use the COPY statement to insert many
# results at relatively high speed.
results_file = results_to_file(results, project_id)
truncate_temp_results()
save_results_to_postgres(results_file, project_id, filter_mode=filter_mode)
except psycopg2.errors.ForeignKeyViolation as e:
sentry.capture_exception(e)
sentry.capture_message(
"could not transfer results to postgres due to ForeignKeyViolation: "
f"{project_id}; filter_mode={filter_mode}"
)
logger.exception(e)
logger.warning(
"could not transfer results to postgres due to ForeignKeyViolation: "
f"{project_id}; filter_mode={filter_mode}"
)
# There is an exception where additional invalid tasks are in a group.
# If that happens we arrive here and add the flag filtermode=true
# to this function, which could solve the issue in save_results_to_postgres.
# If it does not solve the issue we arrive again but
# since filtermode is already true, we will not try to transfer results again.
if not filter_mode:
transfer_results_for_project(project_id, results, filter_mode=True)
except Exception as e:
sentry.capture_exception(e)
sentry.capture_message(f"could not transfer results to postgres: {project_id}")
logger.exception(e)
logger.warning(f"could not transfer results to postgres: {project_id}")
else:
# It is important here that we first insert results into postgres
# and then delete these results from Firebase.
# In case something goes wrong during the insert, results in Firebase
# will not get deleted.
delete_results_from_firebase(project_id, results)
logger.info(f"{project_id}: Transferred results to postgres")
