def index(manager_id: str = None):
api = pillar_api()
if current_user.is_authenticated:
params = {'where': {'owner': {'$in': current_user.groups}}}
else:
params = None
managers = Manager.all(params=params, api=api)
if not manager_id and managers['_items']:
manager_id = managers['_items'][0]._id
manager_limit_reached = managers['_meta'][
'total'] >= flamenco.auth.MAX_MANAGERS_PER_USER
# TODO Sybren: move this to a utility function + check on endpoint to create manager
may_use_flamenco = current_user.has_cap('flamenco-use')
can_create_manager = may_use_flamenco and (not manager_limit_reached or
current_user.has_cap('admin'))
return render_template('flamenco/managers/index.html',
manager_limit_reached=manager_limit_reached,
may_use_flamenco=may_use_flamenco,
can_create_manager=can_create_manager,
max_managers=flamenco.auth.MAX_MANAGERS_PER_USER,
managers=managers,
open_manager_id=manager_id)
def user_is_owner(self,
*,
mngr_doc_id: bson.ObjectId = None,
mngr_doc: dict = None) -> bool:
"""Returns True iff the current user is an owner of the given Flamenco Manager."""
user_id = current_user.user_id
if not current_user.has_cap('flamenco-view'):
self._log.debug(
'user_is_owner(...): user %s does not have flamenco-view cap',
user_id)
return False
if not current_user.has_cap('flamenco-use'):
self._log.debug(
'user_is_owner(...): user %s does not have flamenco-use cap',
user_id)
return False
mngr_doc_id, mngr_doc = self._get_manager(mngr_doc_id, mngr_doc,
{'owner': 1})
owner_group = mngr_doc.get('owner')
if not owner_group:
self._log.warning('user_is_owner(%s): Manager has no owner!',
mngr_doc_id)
return False
user_groups = current_user.get('groups', set())
return owner_group in user_groups
def check_job_permission_fetch_resource(response):
from functools import lru_cache
if current_flamenco.auth.current_user_is_flamenco_admin():
return
if not current_flamenco.manager_manager.user_is_manager():
# Subscribers can read Flamenco jobs.
if current_user.has_cap('flamenco-view'):
return
raise wz_exceptions.Forbidden()
@lru_cache(32)
def user_managers(mngr_doc_id):
return current_flamenco.manager_manager.user_manages(mngr_doc_id=mngr_doc_id)
items = response['_items']
to_remove = []
for idx, job_doc in enumerate(items):
if not user_managers(job_doc.get('manager')):
to_remove.append(idx)
for idx in reversed(to_remove):
del items[idx]
response['_meta']['total'] -= len(items)
def view_job(project, flamenco_props, job_id):
if not request.is_xhr:
return for_project(project, job_id=job_id)
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
from .sdk import Job
from ..managers.sdk import Manager
api = pillar_api()
job = Job.find(job_id, api=api)
try:
manager = Manager.find(job.manager, api=api)
except pillarsdk.ForbiddenAccess:
# It's very possible that the user doesn't have access to this Manager.
manager = None
except pillarsdk.ResourceNotFound:
log.warning('Flamenco job %s has a non-existant manager %s', job_id,
job.manager)
manager = None
from . import (CANCELABLE_JOB_STATES, REQUEABLE_JOB_STATES,
RECREATABLE_JOB_STATES, ARCHIVE_JOB_STATES,
ARCHIVEABLE_JOB_STATES, FAILED_TASKS_REQUEABLE_JOB_STATES)
auth = current_flamenco.auth
write_access = auth.current_user_may(auth.Actions.USE,
bson.ObjectId(project['_id']))
status = job['status']
is_archived = status in ARCHIVE_JOB_STATES
archive_available = is_archived and job.archive_blob_name
# Sort job settings so we can iterate over them in a deterministic way.
job_settings = collections.OrderedDict(
(key, job.settings[key])
for key in sorted(job.settings.to_dict().keys()))
return render_template(
'flamenco/jobs/view_job_embed.html',
job=job,
manager=manager,
project=project,
flamenco_props=flamenco_props.to_dict(),
flamenco_context=request.args.get('context'),
can_cancel_job=write_access and status in CANCELABLE_JOB_STATES,
can_requeue_job=write_access and status in REQUEABLE_JOB_STATES,
can_recreate_job=write_access and status in RECREATABLE_JOB_STATES,
can_archive_job=write_access and status in ARCHIVEABLE_JOB_STATES,
# TODO(Sybren): check that there are actually failed tasks before setting to True:
can_requeue_failed_tasks=write_access
and status in FAILED_TASKS_REQUEABLE_JOB_STATES,
is_archived=is_archived,
write_access=write_access,
archive_available=archive_available,
job_settings=job_settings,
)
def view_task(project, flamenco_props, task_id):
from flamenco.tasks.sdk import Task
api = pillar_api()
if not request.is_xhr:
# Render page that'll perform the XHR.
from flamenco.jobs import routes as job_routes
task = Task.find(task_id, {'projection': {'job': 1}}, api=api)
return job_routes.for_project(project,
job_id=task['job'],
task_id=task_id)
# Task list is public, task details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
task = Task.find(task_id, api=api)
from . import REQUEABLE_TASK_STATES
project_id = bson.ObjectId(project['_id'])
write_access = current_flamenco.auth.current_user_may(
Actions.USE, project_id)
can_requeue_task = write_access and task['status'] in REQUEABLE_TASK_STATES
return render_template('flamenco/tasks/view_task_embed.html',
task=task,
project=project,
flamenco_props=flamenco_props.to_dict(),
flamenco_context=request.args.get('context'),
can_view_log=write_access,
can_requeue_task=can_requeue_task)
def iframe(context: typing.Any, content: str, pargs: typing.List[str],
kwargs: typing.Dict[str, str]) -> str:
"""Show an iframe to users with the required capability.
kwargs:
- 'cap': Capability required for viewing.
- others: Turned into attributes for the iframe element.
"""
import xml.etree.ElementTree as ET
from pillar.auth import current_user
cap = kwargs.pop('cap', '')
if cap:
nocap = kwargs.pop('nocap', '')
if not current_user.has_cap(cap):
if not nocap:
return ''
html = html_module.escape(nocap)
return f'<p class="shortcode nocap">{html}</p>'
kwargs['class'] = f'shortcode {kwargs.get("class", "")}'.strip()
element = ET.Element('iframe', kwargs)
html = ET.tostring(element,
encoding='unicode',
method='html',
short_empty_elements=True)
return html
def check_job_permission_fetch_resource(response):
from functools import lru_cache
if current_flamenco.auth.current_user_is_flamenco_admin():
return
if not current_flamenco.manager_manager.user_is_manager():
# Subscribers can read Flamenco jobs.
if current_user.has_cap('flamenco-view'):
return
raise wz_exceptions.Forbidden()
@lru_cache(32)
def user_managers(mngr_doc_id):
return current_flamenco.manager_manager.user_manages(mngr_doc_id=mngr_doc_id)
items = response['_items']
to_remove = []
for idx, job_doc in enumerate(items):
if not user_managers(job_doc.get('manager')):
to_remove.append(idx)
for idx in reversed(to_remove):
del items[idx]
response['_meta']['total'] -= len(items)
def wrapper(*args, **kwargs):
import pillar.auth
current_user = pillar.auth.get_current_user()
if current_user.is_anonymous:
# We don't need to log at a higher level, as this is very common.
# Many browsers first try to see whether authentication is needed
# at all, before sending the password.
log.debug('Unauthenticated access to %s attempted.', func)
if redirect_to_login:
# Redirect using a 303 See Other, since even a POST
# request should cause a GET on the login page.
return redirect(url_for('users.login', next=request.url), 303)
return render_error()
if require_roles and not current_user.matches_roles(require_roles, require_all):
log.info('User %s is authenticated, but does not have required roles %s to '
'access %s', current_user.user_id, require_roles, func)
return render_error()
if require_cap and not current_user.has_cap(require_cap):
log.info('User %s is authenticated, but does not have required capability %s to '
'access %s', current_user.user_id, require_cap, func)
return render_error()
return func(*args, **kwargs)
def user_may_use(self,
*,
mngr_doc_id: bson.ObjectId = None,
mngr_doc: dict = None) -> bool:
"""Returns True iff this user may use this Flamenco Manager.
Usage implies things like requeuing tasks and jobs, creating new jobs, etc.
"""
from flamenco import current_flamenco
# Flamenco Admins always have access.
if current_flamenco.auth.current_user_is_flamenco_admin():
return True
mngr_doc_id, mngr_doc = self._get_manager(mngr_doc_id, mngr_doc, {
'owner': 1,
'user_groups': 1
})
user_groups = set(current_user.group_ids)
owner_group = mngr_doc.get('owner')
if owner_group and owner_group in user_groups:
return True
if not current_user.has_cap('flamenco-use'):
return False
manager_groups = set(mngr_doc.get('user_groups', []))
return bool(user_groups.intersection(manager_groups))
def billing():
"""View the subscription status of a user
"""
from . import store
log.debug('START OF REQUEST')
if current_user.has_role('protected'):
return abort(404) # TODO: make this 403, handle template properly
expiration_date = 'No subscription to expire'
# Classify the user based on their roles and capabilities
cap_subs = current_user.has_cap('subscriber')
if current_user.has_role('demo'):
user_cls = 'demo'
elif not cap_subs and current_user.has_cap('can-renew-subscription'):
# This user has an inactive but renewable subscription.
user_cls = 'subscriber-expired'
elif cap_subs:
if current_user.has_role('subscriber'):
# This user pays for their own subscription. Only in this case do we need to fetch
# the expiration date from the Store.
user_cls = 'subscriber'
store_user = store.fetch_subscription_info(current_user.email)
if store_user is None:
expiration_date = 'Unable to reach Blender Store to check'
else:
expiration_date = store_user['expiration_date'][:10]
elif current_user.has_role('org-subscriber'):
# An organisation pays for this subscription.
user_cls = 'subscriber-org'
else:
# This user gets the subscription cap from somewhere else (like an organisation).
user_cls = 'subscriber-other'
else:
user_cls = 'outsider'
return render_template('users/settings/billing.html',
user_cls=user_cls,
expiration_date=expiration_date,
title='billing')
def view_job_depsgraph(project, job_id):
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
focus_task_id = request.args.get('t', None)
return render_template('flamenco/jobs/depsgraph.html',
job_id=job_id,
project=project,
focus_task_id=focus_task_id)
def view_job_depsgraph(project, job_id):
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
focus_task_id = request.args.get('t', None)
return render_template('flamenco/jobs/depsgraph.html',
job_id=job_id,
project=project,
focus_task_id=focus_task_id)
def check_task_permission_fetch_resource(response):
# TODO: proper permission checking on project level.
if current_flamenco.auth.current_user_is_flamenco_admin():
return
if not current_flamenco.manager_manager.user_is_manager():
# Subscribers can read Flamenco tasks.
if current_user.has_cap('flamenco-view'):
return
raise wz_exceptions.Forbidden()
def check_task_permission_fetch_resource(response):
# TODO: proper permission checking on project level.
if current_flamenco.auth.current_user_is_flamenco_admin():
return
if not current_flamenco.manager_manager.user_is_manager():
# Subscribers can read Flamenco tasks.
if current_user.has_cap('flamenco-view'):
return
raise wz_exceptions.Forbidden()
def index(manager_id: str = None):
api = pillar_api()
if current_user.is_authenticated:
params = {'where': {'owner': {'$in': current_user.groups}}}
else:
params = None
managers = Manager.all(params=params, api=api)
if not manager_id and managers['_items']:
manager_id = managers['_items'][0]._id
manager_limit_reached = managers['_meta'][
'total'] >= flamenco.auth.MAX_MANAGERS_PER_USER
# TODO Sybren: move this to a utility function + check on endpoint to create manager
may_use_flamenco = current_user.has_cap('flamenco-use')
can_create_manager = may_use_flamenco and (not manager_limit_reached or
current_user.has_cap('admin'))
if session.get('flamenco_last_project'):
project = Project(session.get('flamenco_last_project'))
navigation_links = project_navigation_links(project, pillar_api())
extension_sidebar_links = current_app.extension_sidebar_links(project)
else:
project = None
navigation_links = []
extension_sidebar_links = []
return render_template('flamenco/managers/index.html',
manager_limit_reached=manager_limit_reached,
may_use_flamenco=may_use_flamenco,
can_create_manager=can_create_manager,
max_managers=flamenco.auth.MAX_MANAGERS_PER_USER,
managers=managers,
open_manager_id=manager_id,
project=project,
navigation_links=navigation_links,
extension_sidebar_links=extension_sidebar_links)
def strip_link_and_variations(response):
# Check the access level of the user.
capability = current_app.config['FULL_FILE_ACCESS_CAP']
has_full_access = current_user.has_cap(capability)
# Strip all file variations (unless image) and link to the actual file.
if not has_full_access:
response.pop('link', None)
response.pop('link_expires', None)
# Image files have public variations, other files don't.
if not response.get('content_type', '').startswith('image/'):
if response.get('variations') is not None:
response['variations'] = []
def __call__(self, context: typing.Any, content: str,
pargs: typing.List[str], kwargs: typing.Dict[str,
str]) -> str:
from pillar.auth import current_user
cap = kwargs.pop('cap', '')
if cap:
nocap = kwargs.pop('nocap', '')
if not current_user.has_cap(cap):
if not nocap:
return ''
html = html_module.escape(nocap)
return f'<p class="shortcode nocap">{html}</p>'
return self.decorated(context, content, pargs, kwargs)
def users_edit(user_id):
from pillar.auth import UserClass
if not current_user.has_cap('admin'):
return abort(403)
api = system_util.pillar_api()
try:
user = User.find(user_id, api=api)
except sdk_exceptions.ResourceNotFound:
log.warning('Non-existing user %r requested.', user_id)
raise wz_exceptions.NotFound('Non-existing user %r requested.' % user_id)
form = forms.UserEditForm()
if form.validate_on_submit():
_users_edit(form, user, api)
else:
form.roles.data = user.roles
form.email.data = user.email
user_ob = UserClass.construct('', db_user=user.to_dict())
return render_template('users/edit_embed.html', user=user_ob, form=form)
def index(manager_id: str = None):
api = pillar_api()
managers = Manager.all(api=api)
if not manager_id and managers['_items']:
manager_id = managers['_items'][0]._id
manager_limit_reached = managers['_meta'][
'total'] >= flamenco.auth.MAX_MANAGERS_PER_USER
# TODO Sybren: move this to a utility function + check on endpoint to create manager
may_use_flamenco = current_user.has_cap('flamenco-use')
can_create_manager = may_use_flamenco and not manager_limit_reached
return render_template('flamenco/managers/index.html',
manager_limit_reached=manager_limit_reached,
may_use_flamenco=may_use_flamenco,
can_create_manager=can_create_manager,
max_managers=flamenco.auth.MAX_MANAGERS_PER_USER,
managers=managers,
open_manager_id=manager_id)
def view_task(project, flamenco_props, task_id):
api = pillar_api()
if not request.is_xhr:
# Render page that'll perform the XHR.
from flamenco.jobs import routes as job_routes
task = Task.find(task_id, {'projection': {'job': 1}}, api=api)
return job_routes.for_project(project,
job_id=task['job'],
task_id=task_id)
# Task list is public, task details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
task = Task.find(task_id, api=api)
task.parents = task.parents or [] # Make sure it's always iterable.
from . import REQUEABLE_TASK_STATES, CANCELABLE_TASK_STATES
project_id = bson.ObjectId(project['_id'])
write_access = current_flamenco.auth.current_user_may(
Actions.USE, project_id)
can_requeue_task = write_access and task['status'] in REQUEABLE_TASK_STATES
can_requeue_successors = write_access and task['status'] in (
REQUEABLE_TASK_STATES | {'queued'})
can_cancel_task = write_access and task['status'] in CANCELABLE_TASK_STATES
if write_access and task.log:
# Having task.log means the Manager is using the current approach of sending
# the log tail only. Not having it means the Manager is using the deprecated
# approach of sending the entire log, thus it isn't upgraded to 2.2+ yet, and
# thus it doesn't support the logfile endpoint yet.
manager = Manager.find(task.manager, api=api)
log_download_url = urljoin(manager.url,
f'logfile/{task.job}/{task._id}')
else:
log_download_url = ''
# can_... = is actually possible (but maybe should be forced).
can_request_log_file = task['status'] in LOG_UPLOAD_REQUESTABLE_TASK_STATES
# may_... = the regular 'request' button should be shown.
may_request_log_file = False
log_file_download_url = ''
if write_access and task.log_file:
# Having task.log_file means the same as above, and the Manager has sent us
# the compressed log file.
log_file_download_url = url_for(
'flamenco.tasks.perproject.download_task_log_file',
project_url=project['url'],
task_id=task_id)
elif log_download_url:
# This means the Manager supports sending us the log file, but hasn't yet.
may_request_log_file = can_request_log_file
timing_metrics = task.to_dict().get('metrics', {}).get('timing') or {
} # ensure iterability
return render_template(
'flamenco/tasks/view_task_embed.html',
task=task,
project=project,
flamenco_props=flamenco_props.to_dict(),
flamenco_context=request.args.get('context'),
log_download_url=log_download_url,
can_view_log=write_access,
log_file_download_url=log_file_download_url,
can_request_log_file=can_request_log_file,
may_request_log_file=may_request_log_file,
can_requeue_task=can_requeue_task,
can_requeue_task_and_successors=can_requeue_successors,
can_cancel_task=can_cancel_task,
job_status_help=HELP_FOR_STATUS.get(task['status'], ''),
timing_metrics=timing_metrics,
)
def index():
user_id = current_user.user_id
# Fetch available Managers.
man_man = current_flamenco.manager_manager
managers = list(man_man.owned_managers(
current_user.group_ids, {'_id': 1, 'name': 1}))
manager_limit_reached = not current_user.has_cap('admin') and \
len(managers) >= flamenco.auth.MAX_MANAGERS_PER_USER
# Get the query arguments
identifier: str = request.args.get('identifier', '')
return_url: str = request.args.get('return', '')
request_hmac: str = request.args.get('hmac', '')
ident_oid = str2id(identifier)
keys_coll = current_flamenco.db('manager_linking_keys')
# Verify the received identifier and return URL.
key_info = keys_coll.find_one({'_id': ident_oid})
if key_info is None:
log.warning('User %s tries to link a manager with an identifier that cannot be found',
user_id)
return render_template('flamenco/managers/linking/key_not_found.html')
check_hmac(key_info['secret_key'],
f'{identifier}-{return_url}'.encode('ascii'),
request_hmac)
# Only deal with POST data after HMAC verification is performed.
if request.method == 'POST':
manager_id = request.form['manager-id']
if manager_id == 'new':
manager_name = request.form.get('manager-name', '')
if not manager_name:
raise wz_exceptions.UnprocessableEntity('no Manager name given')
if manager_limit_reached:
log.warning('User %s tries to create a manager %r, but their limit is reached.',
user_id, manager_name)
raise wz_exceptions.UnprocessableEntity('Manager count limit reached')
log.info('Creating new Manager named %r for user %s', manager_name, user_id)
account, mngr_doc, token_data = man_man.create_new_manager(manager_name, '', user_id)
manager_oid = mngr_doc['_id']
else:
log.info('Linking existing Manager %r for user %s', manager_id, user_id)
manager_oid = str2id(manager_id)
# Store that this identifier belongs to this ManagerID
update_res = keys_coll.update_one({'_id': ident_oid},
{'$set': {'manager_id': manager_oid}})
if update_res.matched_count != 1:
log.error('Match count was %s when trying to update secret key '
'for Manager %s on behalf of user %s',
update_res.matched_count, manager_oid, user_id)
raise wz_exceptions.InternalServerError('Unable to store manager ID')
log.info('Manager ID %s is stored as belonging to key with identifier %s',
manager_oid, identifier)
# Now redirect the user back to Flamenco Manager in a verifyable way.
msg = f'{identifier}-{manager_oid}'.encode('ascii')
mac = _compute_hash(key_info['secret_key'], msg)
qs = urllib.parse.urlencode({
'hmac': mac,
'oid': str(manager_oid),
})
direct_to = urllib.parse.urljoin(return_url, f'?{qs}')
log.info('Directing user to URL %s', direct_to)
return redirect(direct_to, 307)
return render_template('flamenco/managers/linking/choose_manager.html',
managers=managers,
can_create_manager=not manager_limit_reached)
def check_task_log_permission_fetch(task_log_docs):
if current_user.has_cap('flamenco-view-logs'):
return
raise wz_exceptions.Forbidden()
def current_user_is_flamenco_user(self) -> bool:
"""Returns True iff the current user has Flamenco User role."""
return current_user.has_cap('flamenco-use')
def check_manager_permissions_create(mngr_doc):
if not current_user.has_cap('flamenco-use'):
log.info('Denying access to create Manager to user %s',
current_user.user_id)
raise wz_exceptions.Forbidden()
def view_job_depsgraph_data(project, job_id, focus_task_id=None):
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
import collections
from flask import jsonify
from flamenco.tasks import COLOR_FOR_TASK_STATUS
from pillar.web.utils import last_page_index
# Collect tasks page-by-page. Stored in a dict to prevent duplicates.
tasks = {}
LIMITED_RESULT_COUNT = 8
def query_tasks(extra_where, extra_update, limit_results: bool):
page_idx = 1
added_in_this_query = 0
while True:
task_page = current_flamenco.task_manager.tasks_for_job(
job_id,
page=page_idx,
max_results=LIMITED_RESULT_COUNT if limit_results else 250,
extra_where=extra_where)
for task in task_page._items:
if task._id in tasks:
continue
task = task.to_dict()
task.update(extra_update)
tasks[task['_id']] = task
added_in_this_query += 1
if limit_results and added_in_this_query >= LIMITED_RESULT_COUNT:
break
if page_idx >= last_page_index(task_page._meta):
break
page_idx += 1
if focus_task_id is None:
# Get the top-level tasks as 'focus tasks'.
# TODO: Test for case of multiple top-level tasks.
extra_where = {
'parents': {
'$exists': 0
},
}
else:
# Otherwise just put in the focus task ID; querying like this ensures
# the returned task belongs to the current job.
extra_where = {'_id': focus_task_id}
log.debug('Querying tasks, focused on %s', extra_where)
query_tasks(extra_where, {'_generation': 0}, False)
# Query for the children & parents of these tasks
already_queried_parents = set()
already_queried_children = set()
def add_parents_children(generation: int, is_outside: bool):
nonlocal already_queried_parents
nonlocal already_queried_children
if not tasks:
return
if is_outside:
extra_update = {'_outside': True}
else:
extra_update = {}
parent_ids = {
parent
for task in tasks.values() for parent in task.get('parents', ())
}
# Get the children of these tasks, but only those we haven't queried for already.
query_children = set(tasks.keys()) - already_queried_children
if query_children:
update = {'_generation': generation, **extra_update}
query_tasks({'parents': {
'$in': list(query_children)
}}, update, is_outside)
already_queried_children.update(query_children)
# Get the parents of these tasks, but only those we haven't queried for already.
query_parents = parent_ids - already_queried_parents
if query_parents:
update = {'_generation': -generation, **extra_update}
query_tasks({'_id': {'$in': list(query_parents)}}, update, False)
already_queried_parents.update(query_parents)
# Add parents/children and grandparents/grandchildren.
# This queries too much, but that's ok for now; this is just a debugging tool.
log.debug('Querying first-level family')
add_parents_children(1, False)
log.debug('Querying second-level family')
add_parents_children(2, True)
# nodes and edges are only told apart by (not) having 'source' and 'target' properties.
graph_items = []
roots = []
xpos_per_generation = collections.defaultdict(int)
for task in sorted(tasks.values(), key=lambda task: task['priority']):
gen = task['_generation']
xpos = xpos_per_generation[gen]
xpos_per_generation[gen] += 1
graph_items.append({
'group': 'nodes',
'data': {
'id': task['_id'],
'label': task['name'],
'status': task['status'],
'color': COLOR_FOR_TASK_STATUS[task['status']],
'outside': task.get('_outside', False),
'focus': task['_id'] == focus_task_id,
},
'position': {
'x': xpos * 100,
'y': gen * -100
},
})
if task.get('parents'):
for parent in task['parents']:
# Skip edges to tasks that aren't even in the graph.
if parent not in tasks: continue
graph_items.append({
'group': 'edges',
'data': {
'id': '%s-%s' % (task['_id'], parent),
'target': task['_id'],
'source': parent,
}
})
else:
roots.append(task['_id'])
return jsonify(elements=graph_items, roots=roots)
def view_job(project, flamenco_props, job_id):
if not request.is_xhr:
return for_project(project, job_id=job_id)
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
from .sdk import Job
from ..managers.sdk import Manager
api = pillar_api()
job = Job.find(job_id, api=api)
try:
manager = Manager.find(job.manager, api=api)
except pillarsdk.ForbiddenAccess:
# It's very possible that the user doesn't have access to this Manager.
manager = None
except pillarsdk.ResourceNotFound:
log.warning('Flamenco job %s has a non-existant manager %s', job_id,
job.manager)
manager = None
users_coll = current_app.db('users')
user = users_coll.find_one(bson.ObjectId(job.user),
projection={
'username': 1,
'full_name': 1
})
if user:
username = user.get('username', '')
full_name = user.get('full_name', '')
user_name = f'{full_name} (@{username})'.strip() or '-unknown-'
else:
user_name = '-unknown-'
from . import (CANCELABLE_JOB_STATES, REQUEABLE_JOB_STATES,
RECREATABLE_JOB_STATES, ARCHIVE_JOB_STATES,
ARCHIVEABLE_JOB_STATES, FAILED_TASKS_REQUEABLE_JOB_STATES)
auth = current_flamenco.auth
write_access = auth.current_user_may(auth.Actions.USE,
bson.ObjectId(project['_id']))
status = job['status']
is_archived = status in ARCHIVE_JOB_STATES
archive_available = is_archived and job.archive_blob_name
# Sort job settings so we can iterate over them in a deterministic way.
job_settings = collections.OrderedDict(
(key, job.settings[key])
for key in sorted(job.settings.to_dict().keys()))
change_prio_states = RECREATABLE_JOB_STATES | REQUEABLE_JOB_STATES | CANCELABLE_JOB_STATES
return render_template(
'flamenco/jobs/view_job_embed.html',
job=job,
user_name=user_name,
manager=manager.to_dict(),
project=project,
flamenco_props=flamenco_props.to_dict(),
flamenco_context=request.args.get('context'),
can_cancel_job=write_access and status in CANCELABLE_JOB_STATES,
can_requeue_job=write_access and status in REQUEABLE_JOB_STATES,
can_recreate_job=write_access and status in RECREATABLE_JOB_STATES,
can_archive_job=write_access and status in ARCHIVEABLE_JOB_STATES,
# TODO(Sybren): check that there are actually failed tasks before setting to True:
can_requeue_failed_tasks=write_access
and status in FAILED_TASKS_REQUEABLE_JOB_STATES,
can_change_prio=write_access and status in change_prio_states,
can_edit_rna_overrides=write_access
and job['job_type'] in blender_render.job_types(),
is_archived=is_archived,
write_access=write_access,
archive_available=archive_available,
job_settings=job_settings,
job_status_help=HELP_FOR_STATUS.get(status, ''),
)
def current_user_is_flamenco_admin(self) -> bool:
"""Returns True iff the user is a Flamenco admin or regular admin."""
return current_user.has_cap('flamenco-admin')
def check_manager_permissions_create(mngr_doc):
if not current_user.has_cap('flamenco-use'):
log.info('Denying access to create Manager to user %s',
current_user.user_id)
raise wz_exceptions.Forbidden()
def view_job(project, flamenco_props, job_id):
if not request.is_xhr:
return for_project(project, job_id=job_id)
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
from .sdk import Job
from ..managers.sdk import Manager
api = pillar_api()
job = Job.find(job_id, api=api)
try:
manager = Manager.find(job.manager, api=api)
except pillarsdk.ForbiddenAccess:
# It's very possible that the user doesn't have access to this Manager.
manager = None
except pillarsdk.ResourceNotFound:
log.warning('Flamenco job %s has a non-existant manager %s', job_id, job.manager)
manager = None
users_coll = current_app.db('users')
user = users_coll.find_one(bson.ObjectId(job.user), projection={'username': 1, 'full_name': 1})
if user:
username = user.get('username', '')
full_name = user.get('full_name', '')
user_name = f'{full_name} (@{username})'.strip() or '-unknown-'
else:
user_name = '-unknown-'
from . import (CANCELABLE_JOB_STATES, REQUEABLE_JOB_STATES, RECREATABLE_JOB_STATES,
ARCHIVE_JOB_STATES, ARCHIVEABLE_JOB_STATES, FAILED_TASKS_REQUEABLE_JOB_STATES)
auth = current_flamenco.auth
write_access = auth.current_user_may(auth.Actions.USE, bson.ObjectId(project['_id']))
status = job['status']
is_archived = status in ARCHIVE_JOB_STATES
archive_available = is_archived and job.archive_blob_name
# Sort job settings so we can iterate over them in a deterministic way.
job_settings = collections.OrderedDict((key, job.settings[key])
for key in sorted(job.settings.to_dict().keys()))
change_prio_states = RECREATABLE_JOB_STATES | REQUEABLE_JOB_STATES | CANCELABLE_JOB_STATES
return render_template(
'flamenco/jobs/view_job_embed.html',
job=job,
user_name=user_name,
manager=manager.to_dict(),
project=project,
flamenco_props=flamenco_props.to_dict(),
flamenco_context=request.args.get('context'),
can_cancel_job=write_access and status in CANCELABLE_JOB_STATES,
can_requeue_job=write_access and status in REQUEABLE_JOB_STATES,
can_recreate_job=write_access and status in RECREATABLE_JOB_STATES,
can_archive_job=write_access and status in ARCHIVEABLE_JOB_STATES,
# TODO(Sybren): check that there are actually failed tasks before setting to True:
can_requeue_failed_tasks=write_access and status in FAILED_TASKS_REQUEABLE_JOB_STATES,
can_change_prio=write_access and status in change_prio_states,
can_edit_rna_overrides=write_access and job['job_type'] in blender_render.job_types(),
is_archived=is_archived,
write_access=write_access,
archive_available=archive_available,
job_settings=job_settings,
job_status_help=HELP_FOR_STATUS.get(status, ''),
)
def sidebar_links(self, project):
if not current_user.has_cap('svn-use'):
return ''
if not self.is_svnman_project(project):
return ''
return flask.render_template('svnman/sidebar.html', project=project)
def process_file(bucket: Bucket, file_id: typing.Union[str, ObjectId],
local_file: tempfile._TemporaryFileWrapper):
"""Process the file by creating thumbnails, sending to Zencoder, etc.
:param file_id: '_id' key of the file
:param local_file: locally stored file, or None if no local processing is
needed.
"""
file_id = ObjectId(file_id)
# Fetch the src_file document from MongoDB.
files = current_app.data.driver.db['files']
src_file = files.find_one(file_id)
if not src_file:
log.warning('process_file(%s): no such file document found, ignoring.')
return
src_file = utils.remove_private_keys(src_file)
# Update the 'format' field from the content type.
# TODO: overrule the content type based on file extention & magic numbers.
mime_category, src_file['format'] = src_file['content_type'].split('/', 1)
# Only allow video encoding when the user has the correct capability.
if not current_user.has_cap('encode-video') and mime_category == 'video':
if src_file['format'].startswith('x-'):
xified = src_file['format']
else:
xified = 'x-' + src_file['format']
src_file['content_type'] = 'application/%s' % xified
mime_category = 'application'
log.info('Not processing video file %s for non-video-encoding user',
file_id)
# Run the required processor, based on the MIME category.
processors: typing.Mapping[str, typing.Callable] = {
'image': _process_image,
'video': _process_video,
}
try:
processor = processors[mime_category]
except KeyError:
log.info(
"POSTed file %s was of type %r, which isn't "
"thumbnailed/encoded.", file_id, mime_category)
src_file['status'] = 'complete'
else:
log.debug('process_file(%s): marking file status as "processing"',
file_id)
src_file['status'] = 'processing'
update_file_doc(file_id, status='processing')
try:
processor(bucket, file_id, local_file, src_file)
except Exception:
log.warning(
'process_file(%s): error when processing file, '
'resetting status to '
'"queued_for_processing"',
file_id,
exc_info=True)
update_file_doc(file_id, status='queued_for_processing')
return
# Update the original file with additional info, e.g. image resolution
r, _, _, status = current_app.put_internal('files', src_file, _id=file_id)
if status not in (200, 201):
log.warning(
'process_file(%s): status %i when saving processed file '
'info to MongoDB: %s', file_id, status, r)
def check_task_log_permission_fetch(task_log_docs):
if current_user.has_cap('flamenco-view-logs'):
return
raise wz_exceptions.Forbidden()
def index():
user_id = current_user.user_id
# Fetch available Managers.
man_man = current_flamenco.manager_manager
manager_cursor, manager_count = man_man.owned_managers(
current_user.group_ids, {
'_id': 1,
'name': 1
})
manager_limit_reached = (not current_user.has_cap('admin')) and \
manager_count >= flamenco.auth.MAX_MANAGERS_PER_USER
# Get the query arguments
identifier: str = request.args.get('identifier', '')
return_url: str = request.args.get('return', '')
request_hmac: str = request.args.get('hmac', '')
ident_oid = str2id(identifier)
keys_coll = current_flamenco.db('manager_linking_keys')
# Verify the received identifier and return URL.
key_info = keys_coll.find_one({'_id': ident_oid})
if key_info is None:
log.warning(
'User %s tries to link a manager with an identifier that cannot be found',
user_id)
return render_template('flamenco/managers/linking/key_not_found.html')
check_hmac(key_info['secret_key'],
f'{identifier}-{return_url}'.encode('ascii'), request_hmac)
# Only deal with POST data after HMAC verification is performed.
if request.method == 'POST':
manager_id = request.form['manager-id']
if manager_id == 'new':
manager_name = request.form.get('manager-name', '')
if not manager_name:
raise wz_exceptions.UnprocessableEntity(
'no Manager name given')
if manager_limit_reached:
log.warning(
'User %s tries to create a manager %r, but their limit is reached.',
user_id, manager_name)
raise wz_exceptions.UnprocessableEntity(
'Manager count limit reached')
log.info('Creating new Manager named %r for user %s', manager_name,
user_id)
account, mngr_doc, token_data = man_man.create_new_manager(
manager_name, '', user_id)
manager_oid = mngr_doc['_id']
else:
log.info('Linking existing Manager %r for user %s', manager_id,
user_id)
manager_oid = str2id(manager_id)
# Store that this identifier belongs to this ManagerID
update_res = keys_coll.update_one(
{'_id': ident_oid}, {'$set': {
'manager_id': manager_oid
}})
if update_res.matched_count != 1:
log.error(
'Match count was %s when trying to update secret key '
'for Manager %s on behalf of user %s',
update_res.matched_count, manager_oid, user_id)
raise wz_exceptions.InternalServerError(
'Unable to store manager ID')
log.info(
'Manager ID %s is stored as belonging to key with identifier %s',
manager_oid, identifier)
# Now redirect the user back to Flamenco Manager in a verifyable way.
msg = f'{identifier}-{manager_oid}'.encode('ascii')
mac = _compute_hash(key_info['secret_key'], msg)
qs = urllib.parse.urlencode({
'hmac': mac,
'oid': str(manager_oid),
})
direct_to = urllib.parse.urljoin(return_url, f'?{qs}')
log.info('Directing user to URL %s', direct_to)
return redirect(direct_to, 307)
return render_template('flamenco/managers/linking/choose_manager.html',
managers=list(manager_cursor),
can_create_manager=not manager_limit_reached)
def view_job_depsgraph_data(project, job_id, focus_task_id=None):
# Job list is public, job details are not.
if not current_user.has_cap('flamenco-view'):
raise wz_exceptions.Forbidden()
import collections
from flask import jsonify
from flamenco.tasks import COLOR_FOR_TASK_STATUS
from pillar.web.utils import last_page_index
# Collect tasks page-by-page. Stored in a dict to prevent duplicates.
tasks = {}
LIMITED_RESULT_COUNT = 8
def query_tasks(extra_where, extra_update, limit_results: bool):
page_idx = 1
added_in_this_query = 0
while True:
task_page = current_flamenco.task_manager.tasks_for_job(
job_id,
page=page_idx,
max_results=LIMITED_RESULT_COUNT if limit_results else 250,
extra_where=extra_where)
for task in task_page._items:
if task._id in tasks:
continue
task = task.to_dict()
task.update(extra_update)
tasks[task['_id']] = task
added_in_this_query += 1
if limit_results and added_in_this_query >= LIMITED_RESULT_COUNT:
break
if page_idx >= last_page_index(task_page._meta):
break
page_idx += 1
if focus_task_id is None:
# Get the top-level tasks as 'focus tasks'.
# TODO: Test for case of multiple top-level tasks.
extra_where = {
'parents': {'$exists': 0},
}
else:
# Otherwise just put in the focus task ID; querying like this ensures
# the returned task belongs to the current job.
extra_where = {'_id': focus_task_id}
log.debug('Querying tasks, focused on %s', extra_where)
query_tasks(extra_where, {'_generation': 0}, False)
# Query for the children & parents of these tasks
already_queried_parents = set()
already_queried_children = set()
def add_parents_children(generation: int, is_outside: bool):
nonlocal already_queried_parents
nonlocal already_queried_children
if not tasks:
return
if is_outside:
extra_update = {'_outside': True}
else:
extra_update = {}
parent_ids = {parent
for task in tasks.values()
for parent in task.get('parents', ())}
# Get the children of these tasks, but only those we haven't queried for already.
query_children = set(tasks.keys()) - already_queried_children
if query_children:
update = {'_generation': generation, **extra_update}
query_tasks({'parents': {'$in': list(query_children)}}, update, is_outside)
already_queried_children.update(query_children)
# Get the parents of these tasks, but only those we haven't queried for already.
query_parents = parent_ids - already_queried_parents
if query_parents:
update = {'_generation': -generation, **extra_update}
query_tasks({'_id': {'$in': list(query_parents)}}, update, False)
already_queried_parents.update(query_parents)
# Add parents/children and grandparents/grandchildren.
# This queries too much, but that's ok for now; this is just a debugging tool.
log.debug('Querying first-level family')
add_parents_children(1, False)
log.debug('Querying second-level family')
add_parents_children(2, True)
# nodes and edges are only told apart by (not) having 'source' and 'target' properties.
graph_items = []
roots = []
xpos_per_generation = collections.defaultdict(int)
for task in sorted(tasks.values(), key=lambda task: task['priority']):
gen = task['_generation']
xpos = xpos_per_generation[gen]
xpos_per_generation[gen] += 1
graph_items.append({
'group': 'nodes',
'data': {
'id': task['_id'],
'label': task['name'],
'status': task['status'],
'color': COLOR_FOR_TASK_STATUS[task['status']],
'outside': task.get('_outside', False),
'focus': task['_id'] == focus_task_id,
},
'position': {'x': xpos * 100, 'y': gen * -100},
})
if task.get('parents'):
for parent in task['parents']:
# Skip edges to tasks that aren't even in the graph.
if parent not in tasks: continue
graph_items.append({
'group': 'edges',
'data': {
'id': '%s-%s' % (task['_id'], parent),
'target': task['_id'],
'source': parent,
}
})
else:
roots.append(task['_id'])
return jsonify(elements=graph_items, roots=roots)
def current_user_is_flamenco_admin(self) -> bool:
"""Returns True iff the user is a Flamenco admin or regular admin."""
return current_user.has_cap('flamenco-admin')
def has_project_settings(self) -> bool:
return current_user.has_cap('svn-use')
def current_user_is_flamenco_user(self) -> bool:
"""Returns True iff the current user has Flamenco User role."""
return current_user.has_cap('flamenco-use')
