class ViewBaseModel(EmptyBaseModel):
"""
This model stores the views / endpoints / resources of the API
This model inherits from :class:`EmptyBaseModel` so it has no traceability
The fields of the model are:
- **id**: int, the primary key of the table, an integer value that is auto incremented
- **name**: str, the name of the view
- **url_rule**: str, the rule for the url
- **description**: the description of the view
"""
__tablename__ = "api_view"
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(128), unique=True, nullable=False)
url_rule = db.Column(db.String(128), nullable=False)
description = db.Column(TEXT, nullable=True)
permissions = db.relationship(
"PermissionViewRoleBaseModel",
backref="api_views",
lazy=True,
primaryjoin=
"and_(ViewBaseModel.id==PermissionViewRoleBaseModel.api_view_id, "
"PermissionViewRoleBaseModel.deleted_at==None)",
cascade="all,delete",
)
def __init__(self, data):
super().__init__()
self.name = data.get("name")
self.url_rule = data.get("url_rule")
self.description = data.get("description")
def __eq__(self, other):
return (isinstance(other, self.__class__)) and (self.name
== other.name)
def __repr__(self):
return f"<View {self.name}>"
def __str__(self):
return self.__repr__()
@classmethod
def get_one_by_name(cls, name: str):
"""
This methods queries the model to search for a view with a given name.
:param str name: The name that the view has
:return: The found result, either an object :class:`ViewBaseModel` or None
:rtype: None or :class:`ViewBaseModel`
"""
return cls.query.filter_by(name=name).first()
class RoleBaseModel(TraceAttributesModel):
"""
This model has the roles that are defined on the REST API
It inherits from :class:`TraceAttributesModel` to have trace fields
The :class:`RoleBaseModel` has the following fields:
- **id**: int, the primary key of the table, an integer value that is auto incremented
- **name**: str, the name of the role
- **created_at**: datetime, the datetime when the user was created (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **updated_at**: datetime, the datetime when the user was last updated (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **deleted_at**: datetime, the datetime when the user was deleted (in UTC).
This field is used only if we deactivate instead of deleting the record.
This datetime is generated automatically, the user does not need to provide it.
"""
__tablename__ = "roles"
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.String(128), nullable=False)
user_roles = db.relationship(
"UserRoleBaseModel",
backref="roles",
lazy=True,
primaryjoin="and_(RoleBaseModel.id==UserRoleBaseModel.role_id, "
"UserRoleBaseModel.deleted_at==None)",
cascade="all,delete",
)
permissions = db.relationship(
"PermissionViewRoleBaseModel",
backref="roles",
lazy=True,
primaryjoin=
"and_(RoleBaseModel.id==PermissionViewRoleBaseModel.role_id, "
"PermissionViewRoleBaseModel.deleted_at==None)",
cascade="all,delete",
)
def __init__(self, data):
super().__init__()
self.id = data.get("id")
self.name = data.get("name")
def __repr__(self):
return f"<Role {self.name}>"
def __str__(self):
return self.__repr__()
class PermissionsDAG(TraceAttributesModel):
__tablename__ = "permission_dag"
__table_args__ = (
{"extend_existing": True},
# db.UniqueConstraint("dag_id", "user_id"),
)
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
dag_id = db.Column(
db.String(128), db.ForeignKey("deployed_dags.id"), nullable=False
)
user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False)
user = db.relationship("UserModel", viewonly=True)
def __init__(self, data):
super().__init__()
self.dag_id = data.get("dag_id")
self.user_id = data.get("user_id")
def __repr__(self):
return f"<DAG permission user: {self.user_id}, DAG: {self.dag_id}<"
@classmethod
def get_user_dag_permissions(cls, user_id):
return cls.query.filter_by(user_id=user_id).all()
@staticmethod
def add_all_permissions_to_user(user_id):
dags = DeployedDAG.get_all_objects()
permissions = [
PermissionsDAG({"dag_id": dag.id, "user_id": user_id}) for dag in dags
]
for permission in permissions:
permission.save()
@staticmethod
def delete_all_permissions_from_user(user_id):
permissions = PermissionsDAG.get_user_dag_permissions(user_id)
for perm in permissions:
perm.delete()
@staticmethod
def check_if_has_permissions(user_id, dag_id):
permission = PermissionsDAG.query.filter_by(
user_id=user_id, dag_id=dag_id
).first()
if permission is None:
return False
return True
class DeployedDAG(TraceAttributesModel):
"""
This model contains the registry of the DAGs that are deployed on the corresponding Airflow server
"""
__tablename__ = "deployed_dags"
id = db.Column(db.String(128), primary_key=True)
description = db.Column(TEXT, nullable=True)
dag_permissions = db.relationship(
"PermissionsDAG",
cascade="all,delete",
backref="deployed_dags",
primaryjoin="and_(DeployedDAG.id==PermissionsDAG.dag_id)",
)
def __init__(self, data):
super().__init__()
self.id = data.get("id")
self.description = data.get("description", None)
def __repr__(self):
return f"<DAG {self.id}>"
class BaseDataModel(TraceAttributesModel):
""" """
__abstract__ = True
data = db.Column(JSON, nullable=True)
checks = db.Column(JSON, nullable=True)
name = db.Column(db.String(256), nullable=False)
description = db.Column(TEXT, nullable=True)
data_hash = db.Column(db.String(256), nullable=False)
schema = db.Column(db.String(256), nullable=True)
@declared_attr
def user_id(self):
return db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False)
@declared_attr
def user(self):
return db.relationship("UserModel")
def __init__(self, data):
self.user_id = data.get("user_id")
self.data = data.get("data") or data.get("execution_results")
self.data_hash = hash_json_256(self.data)
self.name = data.get("name")
self.description = data.get("description")
self.schema = data.get("schema")
self.checks = data.get("checks")
super().__init__()
@classmethod
def get_all_objects(
cls,
user,
schema=None,
creation_date_gte=None,
creation_date_lte=None,
offset=0,
limit=10,
):
"""
Query to get all objects from a user
:param UserModel user: User object.
:param string schema: data_schema to filter (dag)
:param string creation_date_gte: created_at needs to be larger or equal to this
:param string creation_date_lte: created_at needs to be smaller or equal to this
:param int offset: query offset for pagination
:param int limit: query size limit
:return: The objects
:rtype: list(:class:`BaseDataModel`)
"""
query = cls.query.filter(cls.deleted_at == None)
user_access = int(current_app.config["USER_ACCESS_ALL_OBJECTS"])
if (
user is not None
and not user.is_admin()
and not user.is_service_user()
and user_access == 0
):
query = query.filter(cls.user_id == user.id)
if schema:
query = query.filter(cls.schema == schema)
if creation_date_gte:
query = query.filter(cls.created_at >= creation_date_gte)
if creation_date_lte:
query = query.filter(cls.created_at <= creation_date_lte)
# if airflow they also return total_entries = query.count(), for some reason
return query.order_by(desc(cls.created_at)).offset(offset).limit(limit).all()
@classmethod
def get_one_object(cls, user=None, idx=None, **kwargs):
"""
Query to get one object from the user and the id.
:param UserModel user: user object performing the query
:param str or int idx: ID from the object to get
:return: The object or None if it does not exist
:rtype: :class:`BaseDataModel`
"""
user_access = int(current_app.config["USER_ACCESS_ALL_OBJECTS"])
if user is None:
return super().get_one_object(idx=idx)
query = cls.query.filter_by(id=idx, deleted_at=None)
if not user.is_admin() and not user.is_service_user() and user_access == 0:
query = query.filter_by(user_id=user.id)
return query.first()
class ExecutionModel(BaseDataModel):
"""
Model class for the Executions.
It inherits from :class:`BaseDataModel` to have the trace fields and user field
- **id**: str, the primary key for the executions, a hash generated upon creation of the execution
and the id given back to the user.
The hash is generated from the creation date, the user and the id of the parent instance.
- **instance_id**: str, the foreign key for the instance (:class:`InstanceModel`). It links the execution to its
parent instance.
- **name**: str, the name of the execution given by the user.
- **description**: str, the description of the execution given by the user. It is optional.
- **config**: dict (JSON), the configuration to be used in the execution (:class:`ConfigSchema`).
- **data**: dict (JSON), the results from the execution (:class:`DataSchema`).
- **log_text**: text, the log generated by the airflow webserver during execution. This log is stored as text.
- **log_json**: dict (JSON), the log generated by the airflow webserver during execution.
This log is stored as a dict (JSON).
- **user_id**: int, the foreign key for the user (:class:`UserModel`). It links the execution to its owner.
- **created_at**: datetime, the datetime when the execution was created (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **updated_at**: datetime, the datetime when the execution was last updated (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **deleted_at**: datetime, the datetime when the execution was deleted (in UTC). Even though it is deleted,
actually, it is not deleted from the database, in order to have a command that cleans up deleted data
after a certain time of its deletion.
This datetime is generated automatically, the user does not need to provide it.
- **state**: int, value representing state of the execution (finished, in progress, error, etc.)
- **state_message**: str, a string value of state with human readable status message.
- **data_hash**: a hash of the data json using SHA256
:param dict data: the parsed json got from an endpoint that contains all the required information to
create a new execution
"""
# Table name in the database
__tablename__ = "executions"
# Model fields
id = db.Column(db.String(256), nullable=False, primary_key=True)
instance_id = db.Column(db.String(256),
db.ForeignKey("instances.id"),
nullable=False)
config = db.Column(JSON, nullable=False)
dag_run_id = db.Column(db.String(256), nullable=True)
log_text = db.Column(TEXT, nullable=True)
log_json = db.Column(JSON, nullable=True)
state = db.Column(db.SmallInteger,
default=DEFAULT_EXECUTION_CODE,
nullable=False)
state_message = db.Column(
TEXT,
default=EXECUTION_STATE_MESSAGE_DICT[DEFAULT_EXECUTION_CODE],
nullable=True,
)
def __init__(self, data):
super().__init__(data)
self.user_id = data.get("user_id")
self.instance_id = data.get("instance_id")
self.id = hashlib.sha1(
(str(self.created_at) + " " + str(self.user_id) + " " +
str(self.instance_id)).encode()).hexdigest()
self.dag_run_id = data.get("dag_run_id")
self.state = data.get("state", DEFAULT_EXECUTION_CODE)
self.state_message = EXECUTION_STATE_MESSAGE_DICT[self.state]
self.config = data.get("config")
self.log_text = data.get("log_text")
self.log_json = data.get("log_json")
def update_state(self, code, message=None):
"""
Method to update the state code and message of an execution
:param int code: State code for the execution
:param str message: Message for the error
:return: nothing
"""
self.state = code
if message is None:
self.state_message = EXECUTION_STATE_MESSAGE_DICT[code]
else:
self.state_message = message
super().update({})
def __repr__(self):
"""
Method to represent the class :class:`ExecutionModel`
:return: The representation of the :class:`ExecutionModel`
:rtype: str
"""
return f"<Execution {self.id}>"
def __str__(self):
"""
Method to print a string representation of the :class:`ExecutionModel`
:return: The string for the :class:`ExecutionModel`
:rtype: str
"""
return self.__repr__()
class InstanceModel(BaseDataModel):
"""
Model class for the Instances
It inherits from :class:`BaseDataModel` to have the trace fields and user field
The :class:`InstanceModel` has the following fields:
- **id**: str, the primary key for the instances, a hash generated upon creation of the instance
and the id given back to the user.The hash is generated from the creation time and the user id.
- **data**: dict (JSON), the data structure of the instance (:class:`DataSchema`)
- **name**: str, the name given to the instance by the user.
- **description**: str, the description given to the instance by the user. It is optional.
- **executions**: relationship, not a field in the model but the relationship between the _class:`InstanceModel`
and its dependent :class:`ExecutionModel`.
- **user_id**: int, the foreign key for the user (:class:`UserModel`). It links the execution to its owner.
- **created_at**: datetime, the datetime when the instance was created (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **updated_at**: datetime, the datetime when the instance was last updated (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **deleted_at**: datetime, the datetime when the instance was deleted (in UTC). Even though it is deleted,
actually, it is not deleted from the database, in order to have a command that cleans up deleted data
after a certain time of its deletion.
This datetime is generated automatically, the user does not need to provide it.
- **data_hash**: a hash of the data json using SHA256
"""
# Table name in the database
__tablename__ = "instances"
# Model fields
id = db.Column(db.String(256), nullable=False, primary_key=True)
executions = db.relationship(
"ExecutionModel",
backref="instances",
lazy=True,
primaryjoin="and_(InstanceModel.id==ExecutionModel.instance_id, "
"ExecutionModel.deleted_at==None)",
cascade="all,delete",
)
def __init__(self, data):
"""
:param dict data: the parsed json got from an endpoint that contains all the required
information to create a new instance
"""
super().__init__(data)
self.id = hashlib.sha1(
(str(self.created_at) + " " + str(self.user_id)).encode()
).hexdigest()
def __repr__(self):
"""
Method to represent the class :class:`InstanceModel`
:return: The representation of the :class:`InstanceModel`
:rtype: str
"""
return f"<Instance {self.id}>"
def __str__(self):
"""
Method to print a string representation of the :class:`InstanceModel`
:return: The string for the :class:`InstanceModel`
:rtype: str
"""
return self.__repr__()
class UserBaseModel(TraceAttributesModel):
"""
Model class for the Users
It inherits from :class:`TraceAttributesModel` to have trace fields
The :class:`UserBaseModel` has the following fields:
- **id**: int, the primary key for the users, a integer value thar is auto incremented
- **first_name**: str, to store the first name of the user. Usually is an optional field.
- **last_name**: str, to store the last name of the user. Usually is and optional field.
- **username**: str, the username of the user, used for the log in.
- **password**: str, the hashed password stored on the database in the case that the authentication is done
with auth db. In other authentication methods this field is empty
- **email**: str, the email of the user, used to send emails in case of password recovery.
- **created_at**: datetime, the datetime when the user was created (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **updated_at**: datetime, the datetime when the user was last updated (in UTC).
This datetime is generated automatically, the user does not need to provide it.
- **deleted_at**: datetime, the datetime when the user was deleted (in UTC).
This field is used only if we deactivate instead of deleting the record.
This datetime is generated automatically, the user does not need to provide it.
"""
__tablename__ = "users"
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
first_name = db.Column(db.String(128), nullable=True)
last_name = db.Column(db.String(128), nullable=True)
username = db.Column(db.String(128), nullable=False, unique=True)
password = db.Column(db.String(128), nullable=True)
email = db.Column(db.String(128), nullable=False, unique=True)
user_roles = db.relationship(
"UserRoleBaseModel", cascade="all,delete", backref="users"
)
@property
def roles(self):
"""
This property gives back the roles assigned to the user
"""
return {r.role.id: r.role.name for r in self.user_roles}
def __init__(self, data):
super().__init__()
self.first_name = data.get("first_name")
self.last_name = data.get("last_name")
self.username = data.get("username")
# TODO: handle better None passwords that can be found when using ldap
check_pass, msg = check_password_pattern(data.get("password"))
if check_pass:
self.password = self.__generate_hash(data.get("password"))
else:
raise InvalidCredentials(msg)
check_email, msg = check_email_pattern(data.get("email"))
if check_email:
self.email = data.get("email")
else:
raise InvalidCredentials(msg)
def update(self, data):
"""
Updates the user information in the database
:param dict data: the data to update the user
"""
# First we create the hash of the new password and then we update the object
new_password = data.get("password")
if new_password:
new_password = self.__generate_hash(new_password)
data["password"] = new_password
super().update(data)
def comes_from_external_provider(self):
"""
Returns a boolean if the user comes from an external_provider or not
"""
return self.password is None
@staticmethod
def __generate_hash(password):
"""
Method to generate the hash from the password.
:param str password: the password given by the user .
:return: the hashed password.
:rtype: str
"""
if password is None:
return None
return bcrypt.generate_password_hash(password, rounds=10).decode("utf8")
def check_hash(self, password):
"""
Method to check if the hash stored in the database is the same as the password given by the user
:param str password: the password given by the user.
:return: if the password is the same or not.
:rtype: bool
"""
return bcrypt.check_password_hash(self.password, password)
@classmethod
def get_all_users(cls):
"""
Query to get all users
:return: a list with all the users.
:rtype: list(:class:`UserModel`)
"""
return cls.get_all_objects()
@classmethod
def get_one_user(cls, idx):
"""
Query to get the information of one user
:param int idx: ID of the user
:return: the user object
:rtype: :class:`UserModel`
"""
return cls.get_one_object(idx=idx)
@classmethod
def get_one_user_by_email(cls, email):
"""
Query to get one user from the email
:param str email: User email
:return: the user object
:rtype: :class:`UserModel`
"""
return cls.get_one_object(email=email)
@classmethod
def get_one_user_by_username(cls, username):
"""
Returns one user (object) given a username
:param str username: the user username that we want to query for
:return: the user object
:rtype: :class:`UserModel`
"""
return cls.get_one_object(username=username)
def check_username_in_use(self):
"""
Checks if a username is already in use
:return: a boolean if the username is in use
:rtype: bool
"""
return self.query.filter_by(username=self.username).first() is not None
def check_email_in_use(self):
"""
Checks if a email is already in use
:return: a boolean if the username is in use
:rtype: bool
"""
return self.query.filter_by(email=self.email).first() is not None
@staticmethod
def generate_random_password() -> str:
"""
Method to generate a new random password for the user
:return: the newly generated password
:rtype: str
"""
nb_lower = random.randint(1, 9)
nb_upper = random.randint(10 - nb_lower, 11)
nb_numbers = random.randint(1, 3)
nb_special_char = random.randint(1, 3)
upper_letters = random.sample(string.ascii_uppercase, nb_upper)
lower_letters = random.sample(string.ascii_lowercase, nb_lower)
numbers = random.sample(list(map(str, list(range(10)))), nb_numbers)
symbols = random.sample("!¡?¿#$%&'()*+-_./:;,<>=@[]^`{}|~\"\\", nb_special_char)
chars = upper_letters + lower_letters + numbers + symbols
random.shuffle(chars)
pwd = "".join(chars)
return pwd
def is_admin(self) -> bool:
"""
This should return True or False if the user is an admin
:return: if the user is an admin or not
:rtype: bool
"""
return UserRoleBaseModel.is_admin(self.id)
def is_service_user(self) -> bool:
"""
This should return True or False if the user is a service user (type of user used for internal tasks)
:return: if the user is a service_user or not
:rtype: bool
"""
return UserRoleBaseModel.is_service_user(self.id)
def __repr__(self):
"""
Representation method of the class
:return: the representation of the class
:rtype: str
"""
return "<Username {}>".format(self.username)
def __str__(self):
return self.__repr__()
class CaseModel(BaseDataModel):
__tablename__ = "cases"
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
path = db.Column(db.String(500), nullable=False, index=True)
# To find the descendants of this node, we look for nodes whose path
# starts with this node's path.
# c_path =
descendants = db.relationship(
"CaseModel",
viewonly=True,
order_by=path,
primaryjoin=db.remote(db.foreign(path)).like(
path.concat(id).concat(SEPARATOR + "%")),
)
solution = db.Column(JSON, nullable=True)
solution_hash = db.Column(db.String(256), nullable=False)
solution_checks = db.Column(JSON, nullable=True)
# TODO: maybe implement this while making it compatible with sqlite:
# Finding the ancestors is a little bit trickier. We need to create a fake
# secondary table since this behaves like a many-to-many join.
# secondary = select(
# [
# id.label("id"),
# func.unnest(
# cast(
# func.string_to_array(
# func.regexp_replace(path, r"\{}?\d+$".format(SEPARATOR), ""), "."
# ),
# ARRAY(Integer),
# )
# ).label("ancestor_id"),
# ]
# ).alias()
# ancestors = relationship(
# "CaseModel",
# viewonly=True,
# secondary=secondary,
# primaryjoin=id == secondary.c.id,
# secondaryjoin=secondary.c.ancestor_id == id,
# order_by=path,
# )
@property
def depth(self):
return len(self.path.split(SEPARATOR))
def __init__(self, data, parent=None):
super().__init__(data)
if parent is None:
# No parent: we set to empty path
self.path = ""
elif parent.path == "":
# first level has empty path
self.path = str(parent.id) + SEPARATOR
else:
# we compose the path with its parent
self.path = parent.path + str(parent.id) + SEPARATOR
self.solution = data.get("solution", None)
self.solution_hash = hash_json_256(self.solution)
self.solution_checks = data.get("solution_checks", None)
@classmethod
def from_parent_id(cls, user, data):
if data.get("parent_id") is None:
# we assume at root
return cls(data, parent=None)
# we look for the parent object
parent = cls.get_one_object(user=user, idx=data["parent_id"])
if parent is None:
raise ObjectDoesNotExist("Parent does not exist")
if parent.data is not None:
raise InvalidData("Parent cannot be a case")
return cls(data, parent=parent)
def patch(self, data):
"""
Method to patch the case
:param dict data: the patches to apply.
"""
# TODO: review the behaviour of this method.
if "data_patch" in data:
self.data, self.data_hash = self.apply_patch(
self.data, data["data_patch"])
if "solution_patch" in data:
self.solution, self.solution_hash = self.apply_patch(
self.solution, data["solution_patch"])
self.user_id = data.get("user_id")
super().update(data)
def delete(self):
try:
children = [n for n in self.descendants]
for n in children:
db.session.delete(n)
db.session.delete(self)
db.session.commit()
except IntegrityError as e:
db.session.rollback()
log.error(f"Error on deletion of case and children cases: {e}")
except DBAPIError as e:
db.session.rollback()
log.error(
f"Unknown error on deletion of case and children cases: {e}")
@staticmethod
def apply_patch(original_data, data_patch):
"""
Helper method to apply the patch and calculate the new hash
:param dict original_data: the dict with the original data
:param list data_patch: the list with the patch operations to perform
:return: the patched data and the hash
:rtype: Tuple(dict, str)
"""
try:
patched_data = jsonpatch.apply_patch(original_data, data_patch)
except jsonpatch.JsonPatchConflict:
raise InvalidPatch()
except jsonpatch.JsonPointerException:
raise InvalidPatch()
return patched_data, hash_json_256(patched_data)
def move_to(self, new_parent):
new_path = new_parent.path + str(new_parent.id) + SEPARATOR
for n in self.descendants:
n.path = new_path + n.path[len(self.path):]
self.path = new_path
def __repr__(self):
return "<Case {}. Path: {}>".format(self.id, self.path)