def delete(pid: UUID) -> None:
count = ProcessTable.query.filter_by(pid=pid).delete()
db.session.commit()
if count > 0:
return None
else:
raise_status(HTTPStatus.NOT_FOUND)
def subscription_set_in_sync(subscription_id: UUID, current_user: Optional[OIDCUserModel] = Depends(oidc_user)) -> None:
def failed_processes() -> List[str]:
if app_settings.DISABLE_INSYNC_CHECK:
return []
_failed_processes = (
ProcessSubscriptionTable.query.join(ProcessTable)
.filter(ProcessSubscriptionTable.subscription_id == subscription_id)
.filter(~ProcessTable.is_task)
.filter(ProcessTable.last_status != "completed")
.filter(ProcessTable.last_status != "aborted")
.all()
)
return [str(p.pid) for p in _failed_processes]
try:
subscription = get_subscription(subscription_id, for_update=True)
if not subscription.insync:
logger.info(
"Subscription not in sync, trying to change..", subscription_id=subscription_id, user=current_user
)
failed_processes = failed_processes() # type: ignore
if not failed_processes:
subscription.insync = True # type: ignore
db.session.commit()
logger.info("Subscription set in sync", user=current_user)
else:
raise_status(
HTTPStatus.UNPROCESSABLE_ENTITY,
f"Subscription {subscription_id} has still failed processes with id's: {failed_processes}",
)
else:
logger.info("Subscription already in sync")
except ValueError as e:
raise_status(HTTPStatus.NOT_FOUND, str(e))
async def resume_all_processess_endpoint(
request: Request, user: Optional[OIDCUserModel] = Depends(oidc_user)
) -> Dict[str, int]:
"""Retry all task processes in status Failed, Waiting, API Unavailable or Inconsistent Data.
The retry is started in the background, returning status 200 and number of processes in message.
When it is already running, refuse and return status 409 instead.
"""
check_global_lock()
user_name = user.user_name if user else SYSTEM_USER
# Retrieve processes eligible for resuming
processes_to_resume = (ProcessTable.query.filter(
ProcessTable.last_status.in_([
ProcessStatus.FAILED,
ProcessStatus.WAITING,
ProcessStatus.API_UNAVAILABLE,
ProcessStatus.INCONSISTENT_DATA,
])).filter(ProcessTable.is_task.is_(True)).all())
broadcast_func = api_broadcast_process_data(request)
if not await _async_resume_processes(
processes_to_resume, user_name, broadcast_func=broadcast_func):
raise_status(HTTPStatus.CONFLICT,
"Another request to resume all processes is in progress")
logger.info("Resuming all processes", count=len(processes_to_resume))
return {"count": len(processes_to_resume)}
def get_health() -> str:
try:
ProductTable.query.limit(1).with_entities(ProductTable.name)
except OperationalError as e:
logger.warning("Health endpoint returned: notok!")
logger.debug("Health endpoint error details", error=str(e))
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR)
return "OK"
def get_subscription_customer_description_by_customer_subscription(
customer_id: UUID, subscription_id: UUID) -> str:
description = SubscriptionCustomerDescriptionTable.query.filter_by(
customer_id=customer_id,
subscription_id=subscription_id).one_or_none()
if description is None:
raise_status(HTTPStatus.NOT_FOUND)
return description
def subscription_workflows_by_id(subscription_id: UUID) -> Dict[str, List[Dict[str, Union[List[Any], str]]]]:
subscription = SubscriptionTable.query.options(joinedload("product"), joinedload("product.workflows")).get(
subscription_id
)
if not subscription:
raise_status(HTTPStatus.NOT_FOUND)
return subscription_workflows(subscription)
def product_by_id(product_id: UUID) -> ProductTable:
product = (ProductTable.query.options(
joinedload("fixed_inputs"), joinedload("product_blocks"),
joinedload("workflows")).filter_by(product_id=product_id).first())
if not product:
raise_status(HTTPStatus.NOT_FOUND,
f"Product id {product_id} not found")
return product
def start_process(
workflow_key: str,
user_inputs: Optional[List[State]] = None,
user: str = SYSTEM_USER,
broadcast_func: Optional[BroadcastFunc] = None,
) -> Tuple[UUID, Future]:
"""Start a process for workflow.
Args:
workflow_key: name of workflow
user_inputs: List of form inputs from frontend
user: User who starts this process
broadcast_func: Optional function to broadcast process data
Returns:
process id
"""
# ATTENTION!! When modifying this function make sure you make similar changes to `run_workflow` in the test code
if user_inputs is None:
user_inputs = [{}]
pid = uuid4()
workflow = get_workflow(workflow_key)
if not workflow:
raise_status(HTTPStatus.NOT_FOUND, "Workflow does not exist")
initial_state = {
"process_id": pid,
"reporter": user,
"workflow_name": workflow_key,
"workflow_target": workflow.target,
}
try:
state = post_process(workflow.initial_input_form, initial_state,
user_inputs)
except FormValidationError:
logger.exception("Validation errors", user_inputs=user_inputs)
raise
pstat = ProcessStat(pid,
workflow=workflow,
state=Success({
**state,
**initial_state
}),
log=workflow.steps,
current_user=user)
_db_create_process(pstat)
_safe_logstep_withfunc = partial(_safe_logstep,
broadcast_func=broadcast_func)
return _run_process_async(pstat.pid,
lambda: runwf(pstat, _safe_logstep_withfunc))
def product_block_by_id(product_block_id: UUID) -> ProductBlockTable:
product_block = (
ProductBlockTable.query.options(joinedload("resource_types"))
.filter_by(product_block_id=product_block_id)
.first()
)
if not product_block:
raise_status(HTTPStatus.NOT_FOUND, f"Product block id {product_block_id} not found")
return product_block
def delete_product_block(product_block_id: UUID) -> None:
products = ProductTable.query.filter(
ProductTable.product_blocks.any(ProductBlockTable.product_block_id == product_block_id)
).all()
if len(products) > 0:
error_products = ", ".join(map(lambda product: product.name, products))
raise_status(HTTPStatus.BAD_REQUEST, f"ProductBlock {product_block_id} is used in Products: {error_products}")
return delete(ProductBlockTable, product_block_id)
def _get_process(pid: UUID) -> ProcessTable:
process = ProcessTable.query.options(
joinedload(ProcessTable.steps),
joinedload(ProcessTable.process_subscriptions).joinedload(
ProcessSubscriptionTable.subscription),
).get(pid)
if not process:
raise_status(HTTPStatus.NOT_FOUND, f"Process with pid {pid} not found")
return process
def delete_product(product_id: UUID) -> None:
subscriptions = SubscriptionTable.query.filter(
SubscriptionTable.product_id == product_id).all()
if len(subscriptions) > 0:
error_subscriptions = ", ".join(
map(lambda sub: sub.description, subscriptions))
raise_status(
HTTPStatus.BAD_REQUEST,
f"Product {product_id} is used in Subscriptions: {error_subscriptions}"
)
return delete(ProductTable, product_id)
def delete(cls: Type, primary_key: UUID) -> None:
pk = list(
{k: v
for k, v in cls.__table__.columns._collection
if v.primary_key}.keys())[0]
row_count = cls.query.filter(cls.__dict__[pk] == primary_key).delete()
db.session.commit()
if row_count > 0:
return None
else:
raise_status(HTTPStatus.NOT_FOUND)
def delete_subscription(subscription_id: UUID) -> None:
all_process_subscriptions = ProcessSubscriptionTable.query.filter_by(subscription_id=subscription_id).all()
if len(all_process_subscriptions) > 0:
_delete_process_subscriptions(all_process_subscriptions)
return None
else:
subscription = SubscriptionTable.query.filter(SubscriptionTable.subscription_id == subscription_id).first()
if not subscription:
raise_status(HTTPStatus.NOT_FOUND)
_delete_subscription_tree(subscription)
return None
def abort_process_endpoint(
pid: UUID,
request: Request,
user: Optional[OIDCUserModel] = Depends(oidc_user)) -> None:
process = _get_process(pid)
user_name = user.user_name if user else SYSTEM_USER
broadcast_func = api_broadcast_process_data(request)
try:
abort_process(process, user_name, broadcast_func=broadcast_func)
return None
except Exception as e:
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
def update(cls: Type, base_model: BaseModel) -> None:
json_dict = transform_json(base_model.dict())
pk = list(
{k: v
for k, v in cls.__table__.columns._collection
if v.primary_key}.keys())[0]
instance = cls.query.filter(cls.__dict__[pk] == json_dict[pk])
if not instance:
raise_status(HTTPStatus.NOT_FOUND)
json_dict = validate(cls, json_dict, is_new_instance=False)
try:
_merge(cls, json_dict)
except Exception as e:
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
def check_global_lock() -> None:
"""
Check the global lock of the engine.
Returns:
None or raises an exception
"""
engine_settings = EngineSettingsTable.query.one()
if engine_settings.global_lock:
logger.info(
"Unable to interact with processes at this time. Engine StatusEnum is locked"
)
raise_status(
HTTPStatus.SERVICE_UNAVAILABLE,
detail=
"Engine is locked cannot accept changes on processes at this time")
def subscription_instance_in_use_by(
subscription_instance_id: UUID, filter_statuses: List[str] = Depends(_filter_statuses)
) -> List[UUID]:
subscription_instance: SubscriptionInstanceTable = SubscriptionInstanceTable.query.get(subscription_instance_id)
if not subscription_instance:
raise_status(HTTPStatus.NOT_FOUND)
in_use_by_instances = subscription_instance.in_use_by
if filter_statuses:
in_use_by_instances = [sub for sub in in_use_by_instances if sub.subscription.status in filter_statuses]
return list(
filter(
lambda sub_id: sub_id != subscription_instance.subscription_id,
{sub.subscription_id for sub in in_use_by_instances},
)
)
def validate(cls: Type, json_dict: Dict, is_new_instance: bool = True) -> Dict:
required_columns = {
k: v
for k, v in cls.__table__.columns._collection
if not v.nullable and (not v.server_default or v.primary_key)
}
required_attributes: Iterable[str] = required_columns.keys()
if is_new_instance:
required_attributes = filter(
lambda k: not required_columns[k].primary_key, required_attributes)
missing_attributes = list(
filter(lambda key: key not in json_dict, required_attributes))
if len(missing_attributes) != 0:
raise_status(
HTTPStatus.BAD_REQUEST,
f"Missing attributes '{', '.join(missing_attributes)}' for {cls.__name__}",
)
return json_dict
def resume_process_endpoint(
pid: UUID,
request: Request,
json_data: JSON = Body(...),
user: Optional[OIDCUserModel] = Depends(oidc_user)
) -> None:
check_global_lock()
process = _get_process(pid)
if process.last_status == ProcessStatus.RUNNING:
raise_status(HTTPStatus.CONFLICT,
"Resuming a running workflow is not possible")
user_name = user.user_name if user else SYSTEM_USER
broadcast_func = api_broadcast_process_data(request)
resume_process(process,
user=user_name,
user_inputs=json_data,
broadcast_func=broadcast_func)
async def set_global_status(
body: EngineSettingsBaseSchema,
user: Optional[OIDCUserModel] = Depends(oidc_user)
) -> EngineSettingsSchema:
"""
Update the global status of the engine to a new state.
Args:
body: The GlobalStatus object
Returns:
The updated global status object
"""
engine_settings = EngineSettingsTable.query.with_for_update().one()
result = settings.marshall_processes(engine_settings, body.global_lock)
if not result:
raise_status(
status=HTTPStatus.INTERNAL_SERVER_ERROR,
detail=
"Something went wrong while updating the database aborting, possible manual intervention required",
)
if app_settings.SLACK_ENGINE_SETTINGS_HOOK_ENABLED:
user_name = user.user_name if user else SYSTEM_USER
settings.post_update_to_slack(EngineSettingsSchema.from_orm(result),
user_name)
status_response = generate_engine_status_response(result)
if websocket_manager.enabled:
# send engine status to socket.
await websocket_manager.broadcast_data(
[WS_CHANNELS.ENGINE_SETTINGS],
{"engine-status": generate_engine_status_response(result)})
return status_response
def get_subscription_customer_descriptions(_id: UUID) -> str:
description = SubscriptionCustomerDescriptionTable.query.get(_id)
if description is None:
raise_status(HTTPStatus.NOT_FOUND)
return description
def create_or_update(cls: Type, obj: BaseModel) -> None:
try:
json_dict = transform_json(obj.dict())
_merge(cls, json_dict)
except Exception as e:
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
def _query_with_filters(
response: Response,
query: Query,
range: Optional[List[int]] = None,
sort: Optional[List[str]] = None,
filters: Optional[List[str]] = None,
) -> List:
if filters is not None:
for filter in chunked(filters, 2):
if filter and len(filter) == 2:
field = filter[0]
value = filter[1]
value_as_bool = value.lower() in ("yes", "y", "ye", "true",
"1", "ja", "insync")
if value is not None:
if field.endswith("_gt"):
query = query.filter(
SubscriptionTable.__dict__[field[:-3]] > value)
elif field.endswith("_gte"):
query = query.filter(
SubscriptionTable.__dict__[field[:-4]] >= value)
elif field.endswith("_lte"):
query = query.filter(
SubscriptionTable.__dict__[field[:-4]] <= value)
elif field.endswith("_lt"):
query = query.filter(
SubscriptionTable.__dict__[field[:-3]] < value)
elif field.endswith("_ne"):
query = query.filter(
SubscriptionTable.__dict__[field[:-3]] != value)
elif field == "insync":
query = query.filter(
SubscriptionTable.insync.is_(value_as_bool))
elif field == "tags":
# For node and port selector form widgets
sub_values = value.split("-")
query = query.filter(
func.lower(ProductTable.tag).in_(
[s.lower() for s in sub_values]))
elif field == "tag":
# For React table 7
sub_values = value.split("-")
query = query.filter(
func.lower(ProductTable.tag).in_(
[s.lower() for s in sub_values]))
elif field == "product":
sub_values = value.split("-")
query = query.filter(
func.lower(ProductTable.name).in_(
[s.lower() for s in sub_values]))
elif field == "status":
# For React table 7
statuses = value.split("-")
query = query.filter(
SubscriptionTable.status.in_(
[s.lower() for s in statuses]))
elif field == "statuses":
# For port subscriptions
sub_values = value.split("-")
query = query.filter(
SubscriptionTable.status.in_(
[s.lower() for s in sub_values]))
elif field == "organisation":
try:
value_as_uuid = UUID(value)
except (ValueError, AttributeError):
msg = "Not a valid customer_id, must be a UUID: '{value}'"
logger.exception(msg)
raise_status(HTTPStatus.BAD_REQUEST, msg)
query = query.filter(
SubscriptionTable.customer_id == value_as_uuid)
elif field == "tsv":
logger.debug("Running full-text search query.",
value=value)
query = query.search(value)
elif field in SubscriptionTable.__dict__:
query = query.filter(
cast(SubscriptionTable.__dict__[field],
String).ilike("%" + value + "%"))
if sort is not None and len(sort) >= 2:
for item in chunked(sort, 2):
if item and len(item) == 2:
if item[0] in ["product", "tag"]:
field = "name" if item[0] == "product" else "tag"
if item[1].upper() == "DESC":
query = query.order_by(
expression.desc(ProductTable.__dict__[field]))
else:
query = query.order_by(
expression.asc(ProductTable.__dict__[field]))
else:
if item[1].upper() == "DESC":
query = query.order_by(
expression.desc(
SubscriptionTable.__dict__[item[0]]))
else:
query = query.order_by(
expression.asc(
SubscriptionTable.__dict__[item[0]]))
if range is not None and len(range) == 2:
try:
range_start = int(range[0])
range_end = int(range[1])
if range_start >= range_end:
raise ValueError("range start must be lower than end")
except (ValueError, AssertionError):
msg = "Invalid range parameters"
logger.exception(msg)
raise_status(HTTPStatus.BAD_REQUEST, msg)
total = query.count()
query = query.slice(range_start, range_end)
response.headers[
"Content-Range"] = f"subscriptions {range_start}-{range_end}/{total}"
return query.all()
def save(cls: Type, json_data: BaseModel) -> None:
try:
json_dict = transform_json(json_data.dict())
_merge(cls, json_dict)
except Exception as e:
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
def processes_filterable(
response: Response,
range: Optional[str] = None,
sort: Optional[str] = None,
filter: Optional[str] = None,
if_none_match: Optional[str] = Header(None),
) -> List[Dict[str, Any]]:
_range: Union[List[int],
None] = list(map(int, range.split(","))) if range else None
_sort: Union[List[str], None] = sort.split(",") if sort else None
_filter: Union[List[str], None] = filter.split(",") if filter else None
logger.info("processes_filterable() called",
range=_range,
sort=_sort,
filter=_filter)
# the joinedload on ProcessSubscriptionTable.subscription via ProcessBaseSchema.process_subscriptions prevents a query for every subscription later.
# tracebacks are not presented in the list of processes and can be really large.
query = ProcessTable.query.options(
joinedload(ProcessTable.process_subscriptions).joinedload(
ProcessSubscriptionTable.subscription).joinedload(
SubscriptionTable.product),
defer("traceback"),
)
if _filter is not None:
if len(_filter) == 0 or (len(_filter) % 2) > 0:
raise_status(HTTPStatus.BAD_REQUEST,
"Invalid number of filter arguments")
for filter_pair in chunked(_filter, 2):
field, value = filter_pair
field = field.lower()
if value is not None:
if field == "istask":
value_as_bool = value.lower() in ("yes", "y", "ye", "true",
"1", "ja")
query = query.filter(
ProcessTable.is_task.is_(value_as_bool))
elif field == "assignee":
assignees = value.split("-")
query = query.filter(ProcessTable.assignee.in_(assignees))
elif field == "status":
statuses = value.split("-")
query = query.filter(
ProcessTable.last_status.in_(statuses))
elif field == "workflow":
query = query.filter(
ProcessTable.workflow.ilike("%" + value + "%"))
elif field == "creator":
query = query.filter(
ProcessTable.created_by.ilike("%" + value + "%"))
elif field == "organisation":
try:
value_as_uuid = UUID(value)
except (ValueError, AttributeError):
msg = "Not a valid customer_id, must be a UUID: '{value}'"
logger.exception(msg)
raise_status(HTTPStatus.BAD_REQUEST, msg)
process_subscriptions = (
db.session.query(ProcessSubscriptionTable).join(
SubscriptionTable).filter(
SubscriptionTable.customer_id ==
value_as_uuid).subquery())
query = query.filter(
ProcessTable.pid == process_subscriptions.c.pid)
elif field == "product":
process_subscriptions = (
db.session.query(ProcessSubscriptionTable).join(
SubscriptionTable, ProductTable).filter(
ProductTable.name.ilike("%" + value +
"%")).subquery())
query = query.filter(
ProcessTable.pid == process_subscriptions.c.pid)
elif field == "tag":
tags = value.split("-")
process_subscriptions = (
db.session.query(ProcessSubscriptionTable).join(
SubscriptionTable, ProductTable).filter(
ProductTable.tag.in_(tags)).subquery())
query = query.filter(
ProcessTable.pid == process_subscriptions.c.pid)
elif field == "subscriptions":
process_subscriptions = (db.session.query(
ProcessSubscriptionTable
).join(SubscriptionTable).filter(
SubscriptionTable.description.ilike("%" + value +
"%")).subquery())
query = query.filter(
ProcessTable.pid == process_subscriptions.c.pid)
elif field == "pid":
query = query.filter(
cast(ProcessTable.pid,
String).ilike("%" + value + "%"))
elif field == "target":
targets = value.split("-")
process_subscriptions = (
db.session.query(ProcessSubscriptionTable).filter(
ProcessSubscriptionTable.workflow_target.in_(
targets)).subquery())
query = query.filter(
ProcessTable.pid == process_subscriptions.c.pid)
else:
raise_status(HTTPStatus.BAD_REQUEST,
f"Invalid filter '{field}'")
if _sort is not None and len(_sort) >= 2:
for item in chunked(_sort, 2):
if item and len(item) == 2 and item[0] in VALID_SORT_KEYS:
sort_key = VALID_SORT_KEYS[item[0]]
if item[1].upper() == "DESC":
query = query.order_by(
expression.desc(ProcessTable.__dict__[sort_key]))
else:
query = query.order_by(
expression.asc(ProcessTable.__dict__[sort_key]))
else:
raise_status(HTTPStatus.BAD_REQUEST, "Invalid Sort parameters")
if _range is not None and len(_range) == 2:
try:
range_start = int(_range[0])
range_end = int(_range[1])
if range_start >= range_end:
raise ValueError("range start must be lower than end")
except (ValueError, AssertionError):
msg = "Invalid range parameters"
logger.exception(msg)
raise_status(HTTPStatus.BAD_REQUEST, msg)
total = query.count()
query = query.slice(range_start, range_end)
response.headers[
"Content-Range"] = f"processes {range_start}-{range_end}/{total}"
results = query.all()
# Calculate a CRC32 checksum of all the process id's and last_modified_at dates in order as entity tag
checksum = 0
for p in results:
checksum = zlib.crc32(p.pid.bytes, checksum)
last_modified_as_bytes = struct.pack("d",
p.last_modified_at.timestamp())
checksum = zlib.crc32(last_modified_as_bytes, checksum)
entity_tag = hex(checksum)
response.headers["ETag"] = f'W/"{entity_tag}"'
# When the If-None-Match header contains the same CRC we can be sure that the resource has not changed
# so we can skip serialization at the backend and rerendering at the frontend.
if if_none_match == entity_tag:
raise CacheHit(HTTPStatus.NOT_MODIFIED, headers=dict(response.headers))
return [asdict(enrich_process(p)) for p in results]
response.status_code = HTTPStatus.NOT_MODIFIED
return None
response.headers["ETag"] = etag
return model
if cache_response := from_redis(subscription_id):
return _build_response(*cache_response)
try:
subscription_model = SubscriptionModel.from_subscription(subscription_id)
extended_model = build_extendend_domain_model(subscription_model)
etag = _generate_etag(extended_model)
return _build_response(extended_model, etag)
except ValueError as e:
if str(e) == f"Subscription with id: {subscription_id}, does not exist":
raise_status(HTTPStatus.NOT_FOUND, f"Subscription with id: {subscription_id}, not found")
else:
raise_status(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
@router.delete("/{subscription_id}", response_model=None)
def delete_subscription(subscription_id: UUID) -> None:
all_process_subscriptions = ProcessSubscriptionTable.query.filter_by(subscription_id=subscription_id).all()
if len(all_process_subscriptions) > 0:
_delete_process_subscriptions(all_process_subscriptions)
return None
else:
subscription = SubscriptionTable.query.filter(SubscriptionTable.subscription_id == subscription_id).first()
if not subscription:
raise_status(HTTPStatus.NOT_FOUND)
