def get(self, idx):
"""
API method to get the data of the instance that is going to be executed
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by the superuser created for the airflow webserver
:param str idx: ID of the execution
:return: the execution data (body) in a dictionary with structure of :class:`ConfigSchema`
and :class:`DataSchema` and an integer for HTTP status code
:rtype: Tuple(dict, integer)
"""
execution = ExecutionModel.get_one_object(user=self.get_user(),
idx=idx)
if execution is None:
raise ObjectDoesNotExist(error="The execution does not exist")
instance = InstanceModel.get_one_object(user=self.get_user(),
idx=execution.instance_id)
if instance is None:
raise ObjectDoesNotExist(error="The instance does not exist")
config = execution.config
return {
"id": instance.id,
"data": instance.data,
"config": config
}, 200
def put(self, user_id, make_admin):
"""
API method to make admin or take out privileges.
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user. Only an admin can change this.
:param int user_id: id of the user
:param make_admin: 0 if the user is not going to be made admin, 1 if the user has to be made admin
:return: A dictionary with a message (error if authentication failed, or the execution does not exist or
a message) and an integer with the HTTP status code.
:rtype: Tuple(dict, integer)
"""
user_obj = UserModel.get_one_user(user_id)
if user_obj is None:
raise ObjectDoesNotExist()
if make_admin:
UserRoleModel(data={
"user_id": user_id,
"role_id": ADMIN_ROLE
}).save()
else:
UserRoleModel.query.filter_by(user_id=user_id,
role_id=ADMIN_ROLE).delete()
try:
db.session.commit()
except IntegrityError as e:
db.session.rollback()
log.error(f"Integrity error on privileges: {e}")
except DBAPIError as e:
db.session.rollback()
log.error(f"Unknown error on privileges: {e}")
return user_obj, 200
def get_instance_data(instance_id):
instance = InstanceModel.get_one_object(user=user, idx=instance_id)
if instance is None:
raise ObjectDoesNotExist("Instance does not exist")
return dict(data=instance.data,
schema=instance.schema,
checks=instance.checks)
def get_execution_data(execution_id):
execution = ExecutionModel.get_one_object(user=user,
idx=execution_id)
if execution is None:
raise ObjectDoesNotExist("Execution does not exist")
data = get_instance_data(execution.instance_id)
data["solution"] = execution.data
data["solution_checks"] = execution.checks
return data
def from_parent_id(cls, user, data):
if data.get("parent_id") is None:
# we assume at root
return cls(data, parent=None)
# we look for the parent object
parent = cls.get_one_object(user=user, idx=data["parent_id"])
if parent is None:
raise ObjectDoesNotExist("Parent does not exist")
if parent.data is not None:
raise InvalidData("Parent cannot be a case")
return cls(data, parent=parent)
def decorated_func(*args, **kwargs):
"""
The function being decorated
:param args: the arguments of the decorated function
:param kwargs: the keyword arguments of the decorated function
:return: the result of the decorated function or it raises an exception of type :class:`ObjectDoesNotExist`
"""
data = func(*args, **kwargs)
if data is None:
raise ObjectDoesNotExist()
return data
def get(self, idx1, idx2, **kwargs):
"""
API method to generate the json patch of two cases given by the user
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user
:param int idx1: ID of the base case for the comparison
:param int idx2: ID of the case that has to be compared
:return:an object with the instance or instance and execution ID that have been created and the status code
:rtype: Tuple (dict, integer)
"""
if idx1 == idx2:
raise InvalidData("The case identifiers should be different", 400)
case_1 = self.model.get_one_object(user=self.get_user(), idx=idx1)
case_2 = self.model.get_one_object(user=self.get_user(), idx=idx2)
if case_1 is None:
raise ObjectDoesNotExist(
"You don't have access to the first case or it doesn't exist")
elif case_2 is None:
raise ObjectDoesNotExist(
"You don't have access to the second case or it doesn't exist")
elif case_1.schema != case_2.schema:
raise InvalidData(
"The cases asked to compare do not share the same schema")
data = kwargs.get("data", True)
solution = kwargs.get("solution", True)
payload = dict()
if data:
payload["data_patch"] = jsonpatch.make_patch(
case_1.data, case_2.data).patch
if solution:
payload["solution_patch"] = jsonpatch.make_patch(
case_1.solution, case_2.solution).patch
payload["schema"] = case_1.schema
log.info(f"User {self.get_user()} compared cases {idx1} and {idx2}")
return payload, 200
def post(self, idx):
execution = ExecutionModel.get_one_object(user=self.get_user(),
idx=idx)
if execution is None:
raise ObjectDoesNotExist()
af_client = Airflow.from_config(current_app.config)
if not af_client.is_alive():
raise AirflowError(error="Airflow is not accessible")
response = af_client.set_dag_run_to_fail(
dag_name=execution.schema, dag_run_id=execution.dag_run_id)
execution.update_state(EXEC_STATE_STOPPED)
log.info(f"User {self.get_user()} stopped execution {idx}")
return {"message": "The execution has been stopped"}, 200
def get(self, idx):
"""
API method to get the status of the execution created by the user
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user.
:param str idx: ID of the execution
:return: A dictionary with a message (error if the execution does not exist or status of the execution)
and an integer with the HTTP status code.
:rtype: Tuple(dict, integer)
"""
execution = self.data_model.get_one_object(user=self.get_user(),
idx=idx)
if execution is None:
raise ObjectDoesNotExist()
if execution.state not in [EXEC_STATE_RUNNING, EXEC_STATE_UNKNOWN]:
# we only care on asking airflow if the status is unknown or is running.
return execution, 200
def _raise_af_error(execution, error, state=EXEC_STATE_UNKNOWN):
message = EXECUTION_STATE_MESSAGE_DICT[state]
execution.update_state(state)
raise AirflowError(error=error,
payload=dict(message=message, state=state))
dag_run_id = execution.dag_run_id
if not dag_run_id:
# it's safe to say we will never get anything if we did not store the dag_run_id
_raise_af_error(
execution,
state=EXEC_STATE_ERROR,
error="The execution has no dag_run associated",
)
af_client = Airflow.from_config(current_app.config)
if not af_client.is_alive():
_raise_af_error(execution, "Airflow is not accessible")
try:
# TODO: get the dag_name from somewhere!
response = af_client.get_dag_run_status(dag_name=execution.schema,
dag_run_id=dag_run_id)
except AirflowError as err:
_raise_af_error(execution,
f"Airflow responded with an error: {err}")
data = response.json()
state = AIRFLOW_TO_STATE_MAP.get(data["state"], EXEC_STATE_UNKNOWN)
execution.update_state(state)
return execution, 200
def put(self, idx, **req_data):
"""
API method to write the results of the execution
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by the superuser created for the airflow webserver
:param str idx: ID of the execution
:return: A dictionary with a message (body) and an integer with the HTTP status code
:rtype: Tuple(dict, integer)
"""
solution_schema = req_data.pop("solution_schema", "pulp")
# TODO: the solution_schema maybe we should get it from the created execution_id?
# at least, check they have the same schema-name
# Check data format
data = req_data.get("data")
checks = req_data.get("checks")
if data is None:
# only check format if executions_results exist
solution_schema = None
if solution_schema == "pulp":
validate_and_continue(DataSchema(), data)
elif solution_schema is not None:
config = current_app.config
marshmallow_obj = get_schema(config, solution_schema,
SOLUTION_SCHEMA)
validate_and_continue(marshmallow_obj(), data)
# marshmallow_obj().fields['jobs'].nested().fields['successors']
execution = ExecutionModel.get_one_object(user=self.get_user(),
idx=idx)
if execution is None:
raise ObjectDoesNotExist()
state = req_data.get("state", EXEC_STATE_CORRECT)
new_data = dict(
state=state,
state_message=EXECUTION_STATE_MESSAGE_DICT[state],
# because we do not want to store airflow's user:
user_id=execution.user_id,
)
# newly validated data from marshmallow
if data is not None:
new_data["data"] = data
if checks is not None:
new_data["checks"] = checks
req_data.update(new_data)
execution.update(req_data)
# TODO: is this save necessary?
execution.save()
return {"message": "results successfully saved"}, 200
def delete(self, user_id):
"""
:param int user_id: User id.
:return:
:rtype: Tuple(dict, integer)
"""
if self.get_user_id() != user_id and not self.is_admin():
raise NoPermission()
user_obj = UserModel.get_one_user(user_id)
if user_obj is None:
raise ObjectDoesNotExist()
# Service user can not be deleted
if user_obj.is_service_user():
raise NoPermission()
log.info(f"User {user_obj.id} was deleted by user {self.get_user()}")
return self.delete_detail(idx=user_id)
def delete_detail(self, **kwargs):
"""
Method to DELETE an object from the database
:param kwargs: the keyword arguments to identify the object
:return: a message if everything went well and a status code.
"""
item = self.data_model.get_one_object(**kwargs)
if item is None:
raise ObjectDoesNotExist(
"The data entity does not exist on the database")
if self.dependents is not None:
for element in getattr(item, self.dependents):
element.delete()
item.delete()
return {"message": "The object has been deleted"}, 200
def get_user_from_header(self, headers: Headers = None) -> UserBaseModel:
"""
Gets the user represented by the token that has to be in the request headers.
:param headers: the request headers
:type headers: `Headers`
:return: the user object
:rtype: `UserBaseModel`
"""
if headers is None:
raise InvalidUsage(
"Headers are missing from the request. Authentication was not possible to perform"
)
token = self.get_token_from_header(headers)
data = self.decode_token(token)
user_id = data["user_id"]
user = self.user_model.get_one_user(user_id)
if user is None:
raise ObjectDoesNotExist("User does not exist, invalid token")
return user
def post_list(self, data, trace_field="user_id"):
"""
Method to POST one object
:param dict data: the data to create a new object
:param str trace_field: the field that tracks the used that created the object
:return: the newly created item and a status code
"""
data = dict(data)
data[trace_field] = self.get_user_id()
item = self.data_model(data)
if self.foreign_data is not None:
for fk in self.foreign_data:
owner = self.foreign_data[fk].query.get(getattr(item, fk))
if owner is None:
raise ObjectDoesNotExist()
if self.user.id != owner.user_id:
raise NoPermission()
item.save()
return item, 201
def put_detail(self, data, track_user: bool = True, **kwargs):
"""
Method to PUT one object
:param dict data: a dict with the data used for updating the object
:param bool track_user: a control value if the user has to be updated or not
:param kwargs: the keyword arguments to identify the object
:return: a message if everything went well and a status code.
"""
item = self.data_model.get_one_object(**kwargs)
if item is None:
raise ObjectDoesNotExist(
"The data entity does not exist on the database")
data = dict(data)
if track_user:
user_id = kwargs.get("user").get("id") or self.get_user_id()
data["user_id"] = user_id
item.update(data)
return {"message": "Updated correctly"}, 200
def post(self, idx):
"""
API method to copy the information stored in a case to a new instance
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user
:param int idx: ID of the case that has to be copied to an instance or instance and execution
:return: an object with the instance or instance and execution ID that have been created and the status code
:rtype: Tuple (dict, integer)
"""
case = CaseModel.get_one_object(user=self.get_user(), idx=idx)
if case is None:
raise ObjectDoesNotExist()
schema = case.schema
payload = {
"name": "instance_from_" + case.name,
"description": "Instance created from " + case.description,
"data": case.data,
"schema": case.schema,
}
if schema is None:
return self.post_list(payload)
if schema == "pulp" or schema == "solve_model_dag":
validate_and_continue(DataSchema(), payload["data"])
return self.post_list(payload)
config = current_app.config
marshmallow_obj = get_schema(config, schema)
validate_and_continue(marshmallow_obj(), payload["data"])
response = self.post_list(payload)
log.info(
f"User {self.get_user()} creates instance {response[0].id} from case {idx}"
)
return response
def put(self, user_id, **data):
"""
API method to edit an existing user.
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user. Only admin and service user can edit other users.
:param int user_id: id of the user
:return: A dictionary with a message (error if authentication failed, or the execution does not exist or
a message) and an integer with the HTTP status code.
:rtype: Tuple(dict, integer)
"""
if self.get_user_id() != user_id and not self.is_admin():
raise NoPermission()
user_obj = UserModel.get_one_user(user_id)
if user_obj is None:
raise ObjectDoesNotExist()
# working with a ldap service users cannot be edited.
if (current_app.config["AUTH_TYPE"] == AUTH_LDAP
and user_obj.comes_from_external_provider()):
raise EndpointNotImplemented("To edit a user, go to LDAP server")
# working with an OID provider users can not be edited
if (current_app.config["AUTH_TYPE"] == AUTH_OID
and user_obj.comes_from_external_provider()):
raise EndpointNotImplemented(
"To edit a user, go to the OID provider")
if data.get("password"):
check, msg = check_password_pattern(data.get("password"))
if not check:
raise InvalidCredentials(msg)
if data.get("email"):
check, msg = check_email_pattern(data.get("email"))
if not check:
raise InvalidCredentials(msg)
log.info(f"User {user_id} was edited by user {self.get_user()}")
return self.put_detail(data=data, idx=user_id, track_user=False)
def post(self, **kwargs):
"""
API method to create a new execution linked to an already existing instance
It requires authentication to be passed in the form of a token that has to be linked to
an existing session (login) made by a user
:return: A dictionary with a message (error if authentication failed, error if data is not validated or
the reference_id for the newly created execution if successful) and a integer wit the HTTP status code
:rtype: Tuple(dict, integer)
"""
# TODO: should validation should be done even if the execution is not going to be run?
# TODO: should the schema field be cross valdiated with the instance schema field?
config = current_app.config
if "schema" not in kwargs:
kwargs["schema"] = "solve_model_dag"
# TODO: review the order of these two operations
# Get dag config schema and validate it
marshmallow_obj = get_schema(config, kwargs["schema"], "config")
validate_and_continue(marshmallow_obj(), kwargs["config"])
execution, status_code = self.post_list(data=kwargs)
instance = InstanceModel.get_one_object(user=self.get_user(),
idx=execution.instance_id)
if instance is None:
raise ObjectDoesNotExist(
error="The instance to solve does not exist")
# this allows testing without airflow interaction:
if request.args.get("run", "1") == "0":
execution.update_state(EXEC_STATE_NOT_RUN)
return execution, 201
# We now try to launch the task in airflow
af_client = Airflow.from_config(config)
if not af_client.is_alive():
err = "Airflow is not accessible"
log.error(err)
execution.update_state(EXEC_STATE_ERROR_START)
raise AirflowError(
error=err,
payload=dict(
message=EXECUTION_STATE_MESSAGE_DICT[
EXEC_STATE_ERROR_START],
state=EXEC_STATE_ERROR_START,
),
)
# ask airflow if dag_name exists
schema = execution.schema
schema_info = af_client.get_dag_info(schema)
# Validate that instance and dag_name are compatible
marshmallow_obj = get_schema(config, schema, INSTANCE_SCHEMA)
validate_and_continue(marshmallow_obj(), instance.data)
info = schema_info.json()
if info["is_paused"]:
err = "The dag exists but it is paused in airflow"
log.error(err)
execution.update_state(EXEC_STATE_ERROR_START)
raise AirflowError(
error=err,
payload=dict(
message=EXECUTION_STATE_MESSAGE_DICT[
EXEC_STATE_ERROR_START],
state=EXEC_STATE_ERROR_START,
),
)
try:
response = af_client.run_dag(execution.id, dag_name=schema)
except AirflowError as err:
error = "Airflow responded with an error: {}".format(err)
log.error(error)
execution.update_state(EXEC_STATE_ERROR)
raise AirflowError(
error=error,
payload=dict(
message=EXECUTION_STATE_MESSAGE_DICT[EXEC_STATE_ERROR],
state=EXEC_STATE_ERROR,
),
)
# if we succeed, we register the dag_run_id in the execution table:
af_data = response.json()
execution.dag_run_id = af_data["dag_run_id"]
execution.update_state(EXEC_STATE_RUNNING)
log.info("User {} creates execution {}".format(self.get_user_id(),
execution.id))
return execution, 201
