def create_user(email, username, password):
fb_db = firebaseDB()
try:
user = auth.create_user(email=email,
display_name=username,
password=password)
ref = fb_db.reference(f"v2/users/{user.uid}/")
ref.update({
"username":
username,
"taskContributionCount":
0,
"groupContributionCount":
0,
"projectContributionCount":
0,
"created":
datetime.datetime.utcnow().isoformat()[0:-3] +
"Z", # Store current datetime in milliseconds
})
logger.info(f"created new user: {user.uid}")
return user
except Exception as e:
logger.info(f"could not create new user {email}.")
raise CustomError(e)
def send_progress_notification(project_id: str):
"""Send progress notification to project managers in Slack."""
fb_db = auth.firebaseDB()
progress = fb_db.reference(f"v2/projects/{project_id}/progress").get()
if not progress:
logger.info(
f"could not get progress from firebase for project {project_id}")
elif progress >= 90:
project_name = fb_db.reference(f"v2/projects/{project_id}/name").get()
notification_90_sent = fb_db.reference(
f"v2/projects/{project_id}/notification_90_sent").get()
notification_100_sent = fb_db.reference(
f"v2/projects/{project_id}/notification_100_sent").get()
logger.info(
f"{project_id} - progress: {progress},"
f"notifications: {notification_90_sent} {notification_100_sent}")
if progress >= 90 and not notification_90_sent:
# send notification and set value in firebase
send_slack_message(MessageType.NOTIFICATION_90, project_name,
project_id)
fb_db.reference(
f"v2/projects/{project_id}/notification_90_sent").set(True)
if progress >= 100 and not notification_100_sent:
# send notification and set value in firebase
send_slack_message(MessageType.NOTIFICATION_100, project_name,
project_id)
fb_db.reference(
f"v2/projects/{project_id}/notification_100_sent").set(True)
def renew_team_token(team_id):
"""Create new team in Firebase."""
fb_db = firebaseDB() # noqa E841
try:
# check if team exist in firebase
if not fb_db.reference(f"v2/teams/{team_id}").get():
raise CustomError(f"can't find team in firebase: {team_id}")
# get team name from firebase
team_name = fb_db.reference(f"v2/teams/{team_id}/teamName").get()
# check if reference path is valid
ref = fb_db.reference(f"v2/teams/{team_id}")
if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
raise CustomError(
f"""Given argument resulted in invalid Firebase Realtime Database reference.
{ref.path}""")
# generate new uuid4 token
new_team_token = str(uuid.uuid4())
# set team token in firebase
ref.update({"teamToken": new_team_token})
logger.info(
f"renewed team token: {team_id} - '{team_name}' - {new_team_token}"
)
return new_team_token
except Exception as e:
logger.info(f"could not delete team: {team_id}")
raise CustomError(e)
def save_tutorial(self):
"""Save the tutorial in Firebase."""
tutorial = vars(self)
groups = self.groups
tasks = self.tasks
tutorial.pop("groups", None)
tutorial.pop("tasks", None)
tutorial.pop("raw_tasks", None)
tutorial.pop("examplesFile", None)
tutorial.pop("tutorial_tasks", None)
fb_db = auth.firebaseDB()
ref = fb_db.reference("")
if not self.projectId or self.projectId == "":
raise CustomError(
f"""Given argument resulted in invalid Firebase Realtime Database reference.
Project Id is invalid: {self.projectId}""")
ref.update({
f"v2/projects/{self.projectId}": tutorial,
f"v2/groups/{self.projectId}": groups,
f"v2/tasks/{self.projectId}": tasks,
})
logger.info(f"uploaded tutorial data to firebase for {self.projectId}")
ref = fb_db.reference(f"v2/tutorialDrafts/{self.tutorialDraftId}")
ref.set({})
def delete_team(team_id):
"""Delete team in Firebase."""
# TODO: What is the consequence of this on projects and users
# do we expect that the teamId is removed there as well?
# teamId is removed for users, but not for projects at the moment
fb_db = firebaseDB() # noqa E841
try:
# check if team exist in firebase
if not fb_db.reference(f"v2/teams/{team_id}").get():
raise CustomError(f"can't find team in firebase: {team_id}")
# remove all team members
remove_all_team_members(team_id)
# get team name from firebase
team_name = fb_db.reference(f"v2/teams/{team_id}/teamName").get()
# check if reference path is valid, e.g. if team_id is None
ref = fb_db.reference(f"v2/teams/{team_id}")
if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
raise CustomError(
f"""Given argument resulted in invalid Firebase Realtime Database reference.
{ref.path}""")
# delete team in firebase
ref.delete()
logger.info(f"deleted team: {team_id} - '{team_name}'")
except Exception as e:
logger.info(f"could not delete team: {team_id}")
raise CustomError(e)
def move_project_data_to_v2(project_id):
"""
Copy project information from old path to v2/projects in Firebase.
Add status=archived attribute.
Use Firebase transaction function for this.
"""
# Firebase transaction function
def transfer(current_data):
# we need to add these attributes
# since they are expected for version 2
current_data["status"] = "archived"
current_data["projectType"] = 1
current_data["projectId"] = str(project_id)
current_data["progress"] = current_data.get("progress", 0)
current_data["name"] = current_data.get("name", "unknown")
fb_db.reference("v2/projects/{0}".format(project_id)).set(current_data)
return dict()
fb_db = auth.firebaseDB()
projects_ref = fb_db.reference(f"projects/{project_id}")
try:
projects_ref.transaction(transfer)
logger.info(
f"{project_id}: Transfered project to v2 and delete in old path")
return True
except fb_db.TransactionAbortedError:
logger.exception(f"{project_id}: Firebase transaction"
f"for transferring project failed to commit")
return False
def test_deletion(self):
"""Test if tasks, groups, project and results are deleted."""
delete_project.delete_project([self.project_id])
time.sleep(1)
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
self.assertIsNone(ref.get())
ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
self.assertIsNone(ref.get())
ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
self.assertIsNone(ref.get())
ref = fb_db.reference("v2/projects/{0}".format(self.project_id))
self.assertIsNone(ref.get())
pg_db = auth.postgresDB()
sql_query = "SELECT * FROM tasks WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertEqual(result, [])
sql_query = "SELECT * FROM groups WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertEqual(result, [])
sql_query = "SELECT * FROM projects WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertEqual(result, [])
sql_query = "SELECT * FROM results WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertEqual(result, [])
def test_project_id_equals_none(self):
"""Test for project id which does not exists."""
delete_project.delete_project([None])
time.sleep(5) # Wait for Firebase Functions to complete
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/results")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/tasks")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/groups")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/groupsUsers")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/projects")
self.assertIsNotNone(ref.get(shallow=True))
pg_db = auth.postgresDB()
sql_query = f"SELECT * FROM tasks WHERE project_id = '{self.project_id}'"
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = f"SELECT * FROM groups WHERE project_id = '{self.project_id}'"
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = f"SELECT * FROM projects WHERE project_id = '{self.project_id}'"
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = f"SELECT * FROM results WHERE project_id = '{self.project_id}'"
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
def test_project_id_equals_none(self):
"""Test for project id which does not exists."""
delete_project.delete_project([None])
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/results")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/tasks")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/groups")
self.assertIsNotNone(ref.get(shallow=True))
ref = fb_db.reference("v2/projects")
self.assertIsNotNone(ref.get(shallow=True))
pg_db = auth.postgresDB()
sql_query = "SELECT * FROM tasks WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = "SELECT * FROM groups WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = "SELECT * FROM projects WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
sql_query = "SELECT * FROM results WHERE project_id = '{}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertNotEqual(result, [])
def run_create_tutorials() -> None:
"""Create a tutorial project from provided JSON file."""
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/tutorialDrafts/")
tutorial_drafts = ref.get()
if tutorial_drafts is None:
logger.info("There are no tutorial drafts in firebase.")
return None
for tutorial_draft_id, tutorial_draft in tutorial_drafts.items():
tutorial_draft["tutorialDraftId"] = tutorial_draft_id
project_type = tutorial_draft["projectType"]
project_name = tutorial_draft["name"]
try:
tutorial = ProjectType(project_type).tutorial(tutorial_draft)
tutorial.create_tutorial_groups()
tutorial.create_tutorial_tasks()
tutorial.save_tutorial()
send_slack_message(MessageType.SUCCESS, project_name,
tutorial.projectId)
logger.info(f"Success: Tutorial Creation ({project_name})")
except CustomError:
ref = fb_db.reference(f"v2/tutorialDrafts/{tutorial_draft_id}")
ref.set({})
send_slack_message(MessageType.FAIL, project_name,
tutorial.projectId)
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def get_all_projects_of_type(project_type: int):
"""Get the project ids for active and inactive projects in Firebase DB."""
project_id_list = []
fb_db = firebaseDB()
# we neglect private projects here
# since there are no projects set up in production yet
status_list = ["active", "inactive"]
for status in status_list:
logger.info(f"query {status} projects")
projects = (
fb_db.reference(f"v2/projects/")
.order_by_child("status")
.equal_to(status)
.get()
)
for project_id, data in projects.items():
if (data.get("projectType", 1) == project_type) & (
data.get("tutorialId", None) is None
):
project_id_list.append(project_id)
logger.info(f"got {len(project_id_list)} project from firebase.")
return project_id_list
def add_tutorial_id_to_projects(project_id_list, tutorial_id):
fb_db = firebaseDB()
for project_id in project_id_list:
fb_db.reference(f"v2/projects/{project_id}/tutorialId").set(
tutorial_id)
logger.info(
f"added tutorial id '{tutorial_id}' to project '{project_id}'")
def remove_all_team_members(team_id):
"""Remove teamId attribute for all users of the team."""
fb_db = firebaseDB() # noqa E841
try:
# check if team exist in firebase
if not fb_db.reference(f"v2/teams/{team_id}").get():
raise CustomError(f"can't find team in firebase: {team_id}")
# get team name from firebase
team_name = fb_db.reference(f"v2/teams/{team_id}/teamName").get()
# generate random uuid4 token
team_members = (fb_db.reference("v2/users/").order_by_child(
"teamId").equal_to(team_id).get())
# remove teamId attribute for each members
if not team_members:
logger.info(
f"there are no members of the team {team_id} - '{team_name}'")
else:
for user_id in team_members.keys():
# update data in firebase
ref = fb_db.reference(f"v2/users/{user_id}/")
ref.update({"teamId": None})
logger.info(
f"removed teamId {team_id} - '{team_name}' for user {user_id}"
)
logger.info(
f"removed all team members from team: {team_id} - '{team_name}'"
)
except Exception as e:
logger.info(f"could not create team: {team_name}")
raise CustomError(e)
def update_user_data(user_ids: list = []) -> None:
"""Copies new users from Firebase to Postgres."""
# TODO: On Conflict
fb_db = auth.firebaseDB()
pg_db = auth.postgresDB()
ref = fb_db.reference("v2/users")
last_updated = get_last_updated_timestamp()
if user_ids:
logger.info("Add custom user ids to user list, which will be updated.")
users = {}
for user_id in user_ids:
users[user_id] = ref.child(user_id).get()
elif last_updated:
# Get only new users from Firebase.
query = ref.order_by_child("created").start_at(last_updated)
users = query.get()
if len(users) == 0:
logger.info("there are no new users in Firebase.")
else:
# Delete first user in ordered dict (FIFO).
# This user is already in the database (user.created = last_updated).
users.popitem(last=False)
else:
# Get all users from Firebase.
users = ref.get()
for user_id, user in users.items():
# Convert timestamp (ISO 8601) from string to a datetime object.
try:
created = dt.datetime.strptime(
user["created"].replace("Z", ""), "%Y-%m-%dT%H:%M:%S.%f"
)
except KeyError:
# If user has no "created" attribute set it to current time.
created = dt.datetime.utcnow().isoformat()[0:-3] + "Z"
logger.info(
"user {0} didn't have a 'created' attribute. ".format(user_id)
+ "Set it to '{0}' now.".format(created)
)
username = user.get("username", None)
query_update_user = """
INSERT INTO users (user_id, username, created)
VALUES(%s, %s, %s)
ON CONFLICT (user_id) DO UPDATE
SET username=%s,
created=%s;
"""
data_update_user = [
user_id,
username,
created,
username,
created,
]
pg_db.query(query_update_user, data_update_user)
logger.info("Updated user data in Potgres.")
def transfer_results(project_id_list=None):
"""
Download results from firebase,
saves them to postgres and then deletes the results in firebase.
This is implemented as a transactional operation as described in
the Firebase docs to avoid missing new generated results in
Firebase during execution of this function.
"""
# Firebase transaction function
def transfer(current_results):
if current_results is None:
logger.info(f"{project_id}: No results in Firebase")
return dict()
else:
results_user_id_list = get_user_ids_from_results(current_results)
update_data.update_user_data(results_user_id_list)
results_file = results_to_file(current_results, project_id)
save_results_to_postgres(results_file)
return dict()
fb_db = auth.firebaseDB()
if not project_id_list:
# get project_ids from existing results if no project ids specified
project_id_list = fb_db.reference("v2/results/").get(shallow=True)
if not project_id_list:
project_id_list = []
logger.info(f"There are no results to transfer.")
# get all project ids from postgres,
# we will only transfer results for projects we have there
postgres_project_ids = get_projects_from_postgres()
for project_id in project_id_list:
if project_id not in postgres_project_ids:
logger.info(f"{project_id}: This project is not in postgres. "
f"We will not transfer results")
continue
elif "tutorial" in project_id:
logger.info(f"{project_id}: these are results for a tutorial. "
f"We will not transfer these")
continue
logger.info(f"{project_id}: Start transfering results")
results_ref = fb_db.reference(f"v2/results/{project_id}")
truncate_temp_results()
try:
results_ref.transaction(transfer)
logger.info(f"{project_id}: Transfered results to postgres")
except fb_db.TransactionAbortedError:
logger.exception(f"{project_id}: Firebase transaction for "
f"transfering results failed to commit")
del fb_db
return project_id_list
def run_create_projects():
"""
Create projects from submitted project drafts.
Get project drafts from Firebase.
Create projects with groups and tasks.
Save created projects, groups and tasks to Firebase and Postgres.
"""
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/projectDrafts/")
project_drafts = ref.get()
if project_drafts is None:
logger.info("There are no project drafts in firebase.")
return None
for project_draft_id, project_draft in project_drafts.items():
project_draft["projectDraftId"] = project_draft_id
project_type = project_draft["projectType"]
project_name = project_draft["name"]
try:
# Create a project object using appropriate class (project type).
project = ProjectType(project_type).constructor(project_draft)
# TODO: here the project.geometry attribute is overwritten
# this is super confusing since it's not a geojson anymore
# but this is what we set initially,
# e.g. in tile_map_service_grid/project.py
# project.geometry is set to a list of wkt geometries now
# this can't be handled in postgres,
# postgres expects just a string not an array
# validated_geometries should be called during init already
# for the respective project types
project.geometry = project.validate_geometries()
project.create_groups()
project.calc_required_results()
# Save project and its groups and tasks to Firebase and Postgres.
project.save_project()
send_slack_message(MessageType.SUCCESS, project_name,
project.projectId)
logger.info("Success: Project Creation ({0})".format(project_name))
except CustomError as e:
ref = fb_db.reference(f"v2/projectDrafts/{project_draft_id}")
ref.set({})
# check if project could be initialized
try:
project_id = project.projectId
except UnboundLocalError:
project_id = None
send_slack_message(MessageType.FAIL, project_name, project_id,
str(e))
logger.exception(
"Failed: Project Creation ({0}))".format(project_name))
sentry.capture_exception()
continue
def delete_other_old_data():
"""
Delete old imports, results, announcements in Firebase
"""
fb_db = auth.firebaseDB()
fb_db.reference("imports").set({})
fb_db.reference("results").set({})
fb_db.reference("announcements").set({})
logger.info("deleted old results, imports, announcements")
def test_remove_all_team_members(self):
team_management.remove_all_team_members(self.team_id)
# check if no members
fb_db = firebaseDB()
team_members = (
fb_db.reference(f"v2/users/").order_by_child("teamId").equal_to(
self.team_id).get())
self.assertEqual(len(team_members), 0)
def set_up_team_member():
fb_db = firebaseDB()
team_id = "unittest-team-1234"
user_id = "unittest-team-member-1"
data = {"teamId": team_id}
ref = fb_db.reference(f"v2/users/{user_id}")
ref.set(data)
return user_id
def set_up_team():
fb_db = firebaseDB()
team_id = "unittest-team-1234"
team_name = "unittest-team"
team_token = "12345678-1234-5678-1234-567812345678"
data = {"teamName": team_name, "teamToken": team_token}
ref = fb_db.reference(f"v2/teams/{team_id}")
ref.set(data)
return team_id, team_name, team_token
def transfer_results(project_id_list=None):
"""Transfer results for one project after the other.
Will only trigger the transfer of results for projects
that are defined in the postgres database.
Will not transfer results for tutorials and
for projects which are not set up in postgres.
"""
if project_id_list is None:
# get project_ids from existing results if no project ids specified
fb_db = auth.firebaseDB()
project_id_list = fb_db.reference("v2/results/").get(shallow=True)
if project_id_list is None:
project_id_list = []
logger.info("There are no results to transfer.")
# Get all project ids from postgres.
# We will only transfer results for projects we in postgres.
postgres_project_ids = get_projects_from_postgres()
project_id_list_transfered = []
for project_id in project_id_list:
if project_id not in postgres_project_ids:
logger.info(
f"{project_id}: This project is not in postgres. "
f"We will not transfer results"
)
continue
elif "tutorial" in project_id:
logger.info(
f"{project_id}: these are results for a tutorial. "
f"We will not transfer these"
)
continue
else:
logger.info(f"{project_id}: Start transfer results")
fb_db = auth.firebaseDB()
results_ref = fb_db.reference(f"v2/results/{project_id}")
results = results_ref.get()
del fb_db
transfer_results_for_project(project_id, results)
project_id_list_transfered.append(project_id)
return project_id_list_transfered
def get_old_projects():
"""
Get all projects from Firebase which have been created
before we switched to v2.
"""
fb_db = auth.firebaseDB()
ref = fb_db.reference("projects")
projects = ref.get()
logger.info("got old projects from firebase")
return projects
def update_username(email, username):
fb_db = firebaseDB()
try:
user = auth.get_user_by_email(email)
auth.update_user(user.uid, display_name=username)
ref = fb_db.reference(f"v2/users/{user.uid}/username")
ref.set(username)
logger.info(f"updated username for user {email}: {username}")
except Exception as e:
logger.info(f"could not find user {email} in firebase to update username.")
raise CustomError(e)
def delete_user(email):
fb_db = firebaseDB()
try:
user = auth.get_user_by_email(email)
ref = fb_db.reference(f"v2/users/")
ref.update({user.uid: None})
auth.delete_user(user.uid)
logger.info(f"deleted user {email}")
except Exception as e:
logger.info(f"could not find user {email} in firebase to delete.")
raise CustomError(e)
def set_project_manager_rights(email):
fb_db = firebaseDB() # noqa E841
try:
user = auth.get_user_by_email(email)
auth.set_custom_user_claims(user.uid, {"projectManager": True})
logger.info(f"user {email} has project manager rights.")
except Exception as e:
logger.info(
f"could not find user {email} in firebase to set project manager rights."
)
raise CustomError(e)
def test_renew_team_token(self):
new_team_token = team_management.renew_team_token(self.team_id)
self.assertGreaterEqual(len(new_team_token), 32)
self.assertNotEqual(new_team_token, self.team_token)
# check data in Firebase
fb_db = firebaseDB()
ref = fb_db.reference(f"v2/teams/{self.team_id}/teamToken")
team_token = ref.get()
self.assertEqual(team_token, new_team_token)
def tear_down_team_member(user_id):
fb_db = firebaseDB()
# check if reference path is valid, e.g. if team_id is None
ref = fb_db.reference(f"v2/users/{user_id}")
if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
raise CustomError(
f"""Given argument resulted in invalid Firebase Realtime Database reference.
{ref.path}""")
# delete team in firebase
ref.delete()
def remove_project_manager_rights(email):
fb_db = firebaseDB() # noqa E841
try:
user = auth.get_user_by_email(email)
auth.update_user(user.uid, custom_claims=auth.DELETE_ATTRIBUTE)
logger.info(f"user {email} has no project manager rights.")
except Exception as e:
logger.info(
f"could not find user {email} in firebase to remove project manager rights."
)
raise CustomError(e)
def delete_test_user(user_ids: List) -> None:
for user_id in user_ids:
if not re.match(r"[-a-zA-Z0-9]+", user_id):
raise ValueError(f"Given argument resulted in invalid "
f"Firebase Realtime Database reference. ")
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/users/{0}".format(user_id))
ref.delete()
pg_db = auth.postgresDB()
sql_query = "DELETE FROM users WHERE user_id = '{0}'".format(user_id)
pg_db.query(sql_query)
def test_changes(self):
"""Test if results are deleted from Firebase for given project id."""
transfer_results.transfer_results()
fb_db = auth.firebaseDB()
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
self.assertIsNone(ref.get())
pg_db = auth.postgresDB()
sql_query = "SELECT * FROM results WHERE project_id = '{0}' and user_id = '{0}'".format(
self.project_id)
result = pg_db.retr_query(sql_query)
self.assertIsNotNone(result)