def clear_workflow_status(all=False, days=30):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('playbooks', ['read']))
def __func():
if all:
current_app.running_context.execution_db.session.query(
WorkflowStatus).filter(
or_(WorkflowStatus.status == WorkflowStatusEnum.aborted,
WorkflowStatus.status ==
WorkflowStatusEnum.completed)).delete()
elif days > 0:
delete_date = datetime.datetime.today() - datetime.timedelta(
days=days)
current_app.running_context.execution_db.session.query(
WorkflowStatus).filter(
and_(
WorkflowStatus.status.in_([
WorkflowStatusEnum.aborted,
WorkflowStatusEnum.completed
]),
WorkflowStatus.completed_at <= delete_date)).delete(
synchronize_session=False)
current_app.running_context.execution_db.session.commit()
return None, HTTPStatus.NO_CONTENT
return __func()
def update_scheduled_task():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['update', 'execute']))
@with_task('update', request.get_json()['id'])
def __func(task):
data = request.get_json()
invalid_uuids = validate_uuids(data.get('workflows', []))
if invalid_uuids:
return invalid_uuid_problem(invalid_uuids)
if 'name' in data:
same_name = ScheduledTask.query.filter_by(
name=data['name']).first()
if same_name is not None and same_name.id != data['id']:
return scheduled_task_name_already_exists_problem(
same_name, 'update')
try:
task.update(data)
except InvalidTriggerArgs:
return invalid_scheduler_args_problem
else:
db.session.commit()
return task.as_json(), HTTPStatus.OK
return __func()
def update_scheduler_status():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['update', 'execute']))
def __func():
status = request.get_json()['status']
updated_status = current_app.running_context.scheduler.scheduler.state
if status == "start":
updated_status = current_app.running_context.scheduler.start()
current_app.logger.info(
'Scheduler started. Status {0}'.format(updated_status))
elif status == "stop":
updated_status = current_app.running_context.scheduler.stop()
current_app.logger.info(
'Scheduler stopped. Status {0}'.format(updated_status))
elif status == "pause":
updated_status = current_app.running_context.scheduler.pause()
current_app.logger.info(
'Scheduler paused. Status {0}'.format(updated_status))
elif status == "resume":
updated_status = current_app.running_context.scheduler.resume()
current_app.logger.info(
'Scheduler resumed. Status {0}'.format(updated_status))
return {"status": updated_status}, HTTPStatus.OK
return __func()
def read_available_resource_actions():
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('roles', ['read']))
def __func():
return get_all_available_resource_actions(), HTTPStatus.OK
return __func()
def read_all_app_apis(field_name=None):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('app_apis', ['read']))
def __func():
# TODO: Remove this once connexion can validate enums with openapi3.
if field_name and field_name not in [
'info', 'action_apis', 'condition_apis', 'transform_apis',
'device_apis', 'tags', 'external_docs'
]:
return Problem(HTTPStatus.BAD_REQUEST, 'Could not read app api.',
'{} is not a valid field name.'.format(field_name))
ret = []
for app_name, app_api in redis_cache.hgetall("app-apis").items():
ret.append(
format_full_app_api(json.loads(app_api),
app_name.decode("utf-8")))
if field_name is not None:
default = [] if field_name not in ('info', 'external_docs') else {}
ret = [{
'name': api['name'],
field_name: api.get(field_name, default)
} for api in ret]
return ret, HTTPStatus.OK
return __func()
def read_config_values():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('configuration', ['read']))
def __func():
return __get_current_configuration(), HTTPStatus.OK
return __func()
def get_workflow_status(execution_id):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('playbooks', ['read']))
@with_workflow_status('control', execution_id)
def __func(workflow_status):
return workflow_status.as_json(full_actions=True), HTTPStatus.OK
return __func()
def read_scheduled_task(scheduled_task_id):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['read']))
@with_task('read', scheduled_task_id)
def __func(task):
return task.as_json(), HTTPStatus.OK
return __func()
def get_scheduler_status():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['read']))
def __func():
return {
"status": current_app.running_context.scheduler.scheduler.state
}, HTTPStatus.OK
return __func()
def delete_scheduled_task(scheduled_task_id):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['delete']))
@with_task('delete', scheduled_task_id)
def __func(task):
db.session.delete(task)
db.session.commit()
return None, HTTPStatus.NO_CONTENT
return __func()
def read_all_apps():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('app_apis', ['read']))
def __func():
apps = redis_cache.hgetall("app-apis").keys()
print(apps)
return sorted(str(apps),
key=(lambda app_name: app_name.lower())), HTTPStatus.OK
return __func()
def read_all_scheduled_tasks():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['read']))
def __func():
page = request.args.get('page', 1, type=int)
return [
task.as_json() for task in ScheduledTask.query.paginate(
page, current_app.config['ITEMS_PER_PAGE'], False).items
], HTTPStatus.OK
return __func()
def read_app_api(app_name):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('app_apis', ['read']))
def __func():
api = json.loads(redis_cache.hget("app-apis", app_name))
if api is not None:
return format_full_app_api(api, app_name), HTTPStatus.OK
else:
return app_api_dne_problem(app_name)
return __func()
def execute_workflow():
data = request.get_json()
workflow_id = data['workflow_id']
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('playbooks', ['execute']))
@with_workflow('execute', workflow_id)
def __func(workflow):
if not workflow.is_valid:
return Problem(HTTPStatus.BAD_REQUEST, 'Cannot execute workflow',
'Workflow is invalid')
# Short circuits the multiprocessed executor
workflow = current_app.running_context.execution_db.session.query(
Workflow).filter_by(id=workflow_id).first()
workflow_schema = WorkflowSchema()
workflow = workflow_schema.dump(workflow)
execution_id = str(uuid.uuid4())
workflow_data = {
'execution_id': execution_id,
'id': workflow["id"],
'name': workflow["name"]
}
current_app.running_context.results_sender.handle_event(
workflow=workflow,
sender=workflow_data,
event=WalkoffEvent.WorkflowExecutionPending,
data=data)
message = {
"workflow": workflow,
"workflow_id": str(workflow_id),
"execution_id": execution_id
}
current_app.running_context.cache.lpush(
"workflow-queue",
json.dumps(message)) # self.__box.encrypt(message))
current_app.running_context.results_sender.handle_event(
workflow=workflow,
sender=workflow_data,
event=WalkoffEvent.SchedulerJobExecuted,
data=data)
current_app.logger.info('Executed workflow {0}'.format(workflow_id))
return {'id': execution_id}, HTTPStatus.ACCEPTED
return __func()
def get_all_workflow_status():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('playbooks', ['read']))
def __func():
page = request.args.get('page', 1, type=int)
ret = current_app.running_context.execution_db.session.query(WorkflowStatus). \
order_by(WorkflowStatus.status, WorkflowStatus.started_at.desc()). \
limit(current_app.config['ITEMS_PER_PAGE']). \
offset((page - 1) * current_app.config['ITEMS_PER_PAGE'])
ret = jsonify([workflow_status.as_json() for workflow_status in ret])
return ret, HTTPStatus.OK
return __func()
def control_scheduled_task():
scheduled_task_id = request.get_json()['id']
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['execute']))
@with_task('control', scheduled_task_id)
def __func(task):
action = request.get_json()['action']
if action == 'start':
task.start()
elif action == 'stop':
task.stop()
db.session.commit()
return {}, HTTPStatus.OK
return __func()
def read_app_api_field(app_name, field_name):
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('app_apis', ['read']))
def __func():
# TODO: Remove this once connexion can validate enums with openapi3.
if field_name not in [
'info', 'action_apis', 'condition_apis', 'transform_apis',
'device_apis', 'tags', 'externalDocs'
]:
return Problem(HTTPStatus.BAD_REQUEST, 'Could not read app api.',
'{} is not a valid field name.'.format(field_name))
api = json.loads(redis_cache.hget("app-apis", app_name))
if api is not None:
return format_full_app_api(api,
app_name)[field_name], HTTPStatus.OK
else:
return app_api_dne_problem(app_name)
return __func()
def create_scheduled_task():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['create', 'execute']))
def __func():
data = request.get_json()
invalid_uuids = validate_uuids(data['workflows'])
if invalid_uuids:
return invalid_uuid_problem(invalid_uuids)
task = ScheduledTask.query.filter_by(name=data['name']).first()
if task is None:
try:
task = ScheduledTask(**data)
except InvalidTriggerArgs:
return invalid_scheduler_args_problem
else:
db.session.add(task)
db.session.commit()
return task.as_json(), HTTPStatus.CREATED
else:
return scheduled_task_name_already_exists_problem(
data['name'], 'create')
return __func()
def update_configuration():
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('configuration', ['update']))
def __func():
config_in = request.get_json()
if not _reset_token_durations(access_token_duration=config_in.get(
'access_token_duration', None),
refresh_token_duration=config_in.get(
'refresh_token_duration', None)):
return Problem.from_crud_resource(
HTTPStatus.BAD_REQUEST, 'configuration', 'update',
'Access token duration must be less than refresh token duration.'
)
for config, config_value in config_in.items():
if hasattr(api_gateway.config.Config, config.upper()):
setattr(api_gateway.config.Config, config.upper(),
config_value)
elif hasattr(current_app.config, config.upper()):
setattr(current_app.config, config.upper(), config_value)
current_app.logger.info('Changed configuration')
try:
api_gateway.config.Config.write_values_to_file()
return __get_current_configuration(), HTTPStatus.OK
except (IOError, OSError) as e:
current_app.logger.error(
'Could not write changes to configuration to file')
return Problem(
HTTPStatus.INTERNAL_SERVER_ERROR,
'Could not write changes to file.',
'Could not write configuration changes to file. Problem: {}'.
format(format_exception_message(e)))
return __func()
def control_workflow():
data = request.get_json()
execution_id = data['execution_id']
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('playbooks', ['execute']))
@validate_execution_id_is_registered('control', execution_id)
def __func():
status = data['status']
if status == 'pause':
current_app.running_context.executor.pause_workflow(
execution_id, user=get_jwt_claims().get('username', None))
elif status == 'resume':
current_app.running_context.executor.resume_workflow(
execution_id, user=get_jwt_claims().get('username', None))
elif status == 'abort':
abort_workflow(execution_id,
user=get_jwt_claims().get('username', None))
return None, HTTPStatus.NO_CONTENT
return __func()
def dashboard_getter(dashboard):
if helpers.validate_uuid(dashboard):
return current_app.running_context.execution_db.session.query(
Dashboard).filter_by(id_=dashboard).first()
else:
return current_app.running_context.execution_db.session.query(
Dashboard).filter_by(name=dashboard).first()
dashboard_schema = DashboardSchema()
with_dashboard = with_resource_factory("dashboard", dashboard_getter)
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions("dashboards", ["create"]))
def create_dashboard():
data = request.get_json()
dashboard_name = data["name"]
try:
dashboard = dashboard_schema.load(data)
current_app.running_context.execution_db.session.add(dashboard)
current_app.running_context.execution_db.session.commit()
return dashboard_schema.dump(dashboard), HTTPStatus.CREATED
except ValidationError as e:
current_app.running_context.execution_db.session.rollback()
return improper_json_problem("dashboard", "create", dashboard_name,
e.messages)
except (IntegrityError, StatementError):
current_app.running_context.execution_db.session.rollback()
from flask_jwt_extended import jwt_required
from api_gateway.extensions import db
from api_gateway.security import permissions_accepted_for_resources, ResourcePermissions
from api_gateway.server.decorators import with_resource_factory
from api_gateway.server.problem import Problem
from http import HTTPStatus
from api_gateway.serverdb import clear_resources_for_role, get_all_available_resource_actions
from api_gateway.serverdb.role import Role
with_role = with_resource_factory(
'role', lambda role_id: Role.query.filter_by(id=role_id).first())
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('roles', ['read']))
def read_all_roles():
roles = []
for role in Role.query.all():
# hides internal and super_admin roles
if role.id != 1 and role.id != 2:
roles.append(role.as_json())
return roles, HTTPStatus.OK
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('roles', ['create']))
def create_role():
json_data = request.get_json()
if not Role.query.filter_by(name=json_data['name']).first():
resources = json_data['resources'] if 'resources' in json_data else []
from flask_jwt_extended import jwt_required
import api_gateway.flask_config
from api_gateway.helpers import format_exception_message
from api_gateway.security import permissions_accepted_for_resources, ResourcePermissions
from api_gateway.server.problem import Problem
from http import HTTPStatus
def __get_current_settings():
return {'access_token_duration': int(current_app.config['JWT_ACCESS_TOKEN_EXPIRES'].seconds / 60),
'refresh_token_duration': int(current_app.config['JWT_REFRESH_TOKEN_EXPIRES'].days)}
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('settings', ['read']))
def read_settings():
return __get_current_settings(), HTTPStatus.OK
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('settings', ['update']))
def update_settings():
config_in = request.get_json()
if not _reset_token_durations(access_token_duration=config_in.get('access_token_duration', None),
refresh_token_duration=config_in.get('refresh_token_duration', None)):
return Problem.from_crud_resource(
HTTPStatus.BAD_REQUEST,
'settings',
'update',
'Access token duration must be less than refresh token duration.')
def delete_global(global_var):
global_id = str(global_var.id_)
to_delete = auth_check(global_id, "delete", "global_variables")
if to_delete:
current_app.running_context.execution_db.session.delete(global_var)
current_app.logger.info(f"Global_variable removed {global_var.name}")
current_app.running_context.execution_db.session.commit()
return None, HTTPStatus.NO_CONTENT
else:
return None, HTTPStatus.FORBIDDEN
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions("global_variables", ["create"]))
def create_global():
data = request.get_json()
global_id = data['id_']
new_permissions = data['permissions']
if new_permissions:
update_permissions("global_variables",
global_id,
new_permissions=new_permissions)
else:
default_permissions("global_variables", global_id)
try:
with open(config.ENCRYPTION_KEY_PATH, 'rb') as f:
data['value'] = fernet_encrypt(f.read(), data['value'])
'Scheduled task',
lambda task_id: ScheduledTask.query.filter_by(id=task_id).first())
def validate_uuids(uuids):
invalid_uuids = []
for uuid in uuids:
try:
UUID(uuid)
except ValueError:
invalid_uuids.append(uuid)
return invalid_uuids
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('scheduler', ['read']))
def get_scheduler_status():
return {
"status": current_app.running_context.scheduler.scheduler.state
}, HTTPStatus.OK
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('scheduler', ['update', 'execute']))
def update_scheduler_status():
status = request.get_json()['status']
updated_status = current_app.running_context.scheduler.scheduler.state
if status == "start":
updated_status = current_app.running_context.scheduler.start()
current_app.logger.info(
Workflow).filter_by(name=workflow).first()
with_workflow = with_resource_factory('workflow', workflow_getter)
ALLOWED_EXTENSIONS = {'json'}
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions('workflows', ['create']))
def create_workflow():
data = request.get_json()
workflow_id = request.args.get("source")
workflow_name = data['name']
try:
new_permissions = data['permissions']
except:
new_permissions = []
if request.files and 'file' in request.files:
data = json.loads(request.files['file'].read().decode('utf-8'))
if workflow_id:
wf = current_app.running_context.execution_db.session.query(Workflow)\
curr_user_id = (db.session.query(User).filter(User.username == username).first()).id
global_id = str(global_var.id_)
to_delete = auth_check(global_id, "delete", "global_variables")
if (global_var.creator == curr_user_id) or to_delete:
current_app.running_context.execution_db.session.delete(global_var)
current_app.logger.info(f"Global_variable removed {global_var.name}")
current_app.running_context.execution_db.session.commit()
return None, HTTPStatus.NO_CONTENT
else:
return None, HTTPStatus.FORBIDDEN
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions("global_variables", ["create"]))
def create_global():
data = request.get_json()
global_id = data.get('id_', str(uuid4()))
username = get_jwt_claims().get('username', None)
curr_user = db.session.query(User).filter(User.username == username).first()
data.update({'creator': curr_user.id})
new_permissions = data.get('permissions', None)
access_level = data.get('access_level', 1)
# creator only
if access_level == 0:
update_permissions("global_variables", global_id,
new_permissions=[{"role": 1, "permissions": ["delete", "execute", "read", "update"]}],
from flask_jwt_extended import jwt_required
from api_gateway.extensions import db
from api_gateway.security import permissions_accepted_for_resources, ResourcePermissions, admin_required
from api_gateway.server.decorators import with_resource_factory
from api_gateway.server.problem import Problem
from http import HTTPStatus
from api_gateway.serverdb import clear_resources_for_role, get_all_available_resource_actions
from api_gateway.serverdb.role import Role
with_role = with_resource_factory(
'role', lambda role_id: Role.query.filter_by(id=role_id).first())
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('roles', ['read']))
def read_all_roles():
roles = []
for role in Role.query.all():
if role.id != 2:
roles.append(role.as_json())
return roles, HTTPStatus.OK
@jwt_required
@admin_required
def create_role():
json_data = request.get_json()
if not Role.query.filter_by(name=json_data['name']).first():
resources = json_data['resources'] if 'resources' in json_data else []
if '/roles' in resources:
# current_app.running_context.execution_db.session.commit()
# gevent.spawn(push_to_workflow_stream_queue, workflow_status_json, "PENDING")
# current_app.logger.info(f"Created Workflow Status {workflow.name} ({execution_id})")
# return jsonify({'id': execution_id}), HTTPStatus.ACCEPTED
# except ValidationError as e:
# current_app.running_context.execution_db.session.rollback()
# return improper_json_problem('workflow_status', 'create', workflow.name, e.messages)
# except IntegrityError:
# current_app.running_context.execution_db.session.rollback()
# return unique_constraint_problem('workflow_status', 'create', workflow.name)
# TODO: maybe make an internal user for the worker/umpire?
@jwt_required
@permissions_accepted_for_resources(
ResourcePermissions("workflowstatus", ["create"]))
@with_workflow_status('update', 'execution_id')
def update_workflow_status(execution_id):
old_workflow_status = workflow_status_schema.dump(execution_id)
data = request.get_json()
# TODO: change these on the db model to be keyed by ID
if "node_statuses" in old_workflow_status:
old_workflow_status["node_statuses"] = {
astat['node_id']: astat
for astat in old_workflow_status["node_statuses"]
}
else:
old_workflow_status["node_statuses"] = {}
patch = jsonpatch.JsonPatch.from_string(json.dumps(data))
with_workflow_status = with_resource_factory('workflow',
workflow_status_getter,
validator=is_valid_uid)
status_order = OrderedDict([((StatusEnum.EXECUTING, StatusEnum.AWAITING_DATA,
StatusEnum.PAUSED), WorkflowStatus.started_at),
((StatusEnum.ABORTED, StatusEnum.COMPLETED),
WorkflowStatus.completed_at)])
executing_statuses = (StatusEnum.EXECUTING, StatusEnum.AWAITING_DATA,
StatusEnum.PAUSED)
completed_statuses = (StatusEnum.ABORTED, StatusEnum.COMPLETED)
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('workflows', ['read']))
@paginate(workflow_status_schema) # ToDo: make this summary
def get_all_workflow_status():
r = current_app.running_context.execution_db.session.query(
WorkflowStatus).order_by(WorkflowStatus.name).all()
return r, HTTPStatus.OK
@jwt_required
@permissions_accepted_for_resources(ResourcePermissions('workflows', ['read']))
@with_workflow_status('control', 'execution')
def get_workflow_status(execution):
workflow_status = workflow_status_schema.dump(execution)
return workflow_status, HTTPStatus.OK