def setup_for_flamenco(project: pillarsdk.Project):
from pillar.api.utils import str2id
import flamenco.setup
project_id = project._id
if not project.has_method('PUT'):
log.warning(
'User %s tries to set up project %s for Flamenco, but has no PUT rights.',
current_user, project_id)
raise wz_exceptions.Forbidden()
if not current_flamenco.auth.current_user_is_flamenco_user():
log.warning(
'User %s tries to set up project %s for Flamenco, but is not flamenco-user.',
current_user, project_id)
raise wz_exceptions.Forbidden()
log.info('User %s sets up project %s for Flamenco', current_user,
project_id)
flamenco.setup.setup_for_flamenco(project.url)
# Find the Managers available to this user, so we can auto-assign if there is exactly one.
man_man = current_flamenco.manager_manager
managers = man_man.owned_managers(
[bson.ObjectId(gid) for gid in current_user.groups])
manager_count = managers.count()
project_oid = str2id(project_id)
user_id = current_user_id()
if manager_count == 0:
_, mngr_doc, _ = man_man.create_new_manager('My Manager', '', user_id)
assign_man_oid = mngr_doc['_id']
log.info(
'Created and auto-assigning Manager %s to project %s upon setup for Flamenco.',
assign_man_oid, project_oid)
man_man.api_assign_to_project(assign_man_oid, project_oid, 'assign')
elif manager_count == 1:
assign_manager = managers.next()
assign_man_oid = str2id(assign_manager['_id'])
log.info(
'Auto-assigning Manager %s to project %s upon setup for Flamenco.',
assign_man_oid, project_oid)
man_man.api_assign_to_project(assign_man_oid, project_oid, 'assign')
return '', 204
def view_embed(organization_id: str):
if not request.is_xhr:
return index(organization_id)
api = pillar_api()
organization: Organization = Organization.find(organization_id, api=api)
om = current_app.org_manager
organization_oid = str2id(organization_id)
members = om.org_members(organization.members)
for member in members:
member['avatar'] = gravatar(member.get('email'))
member['_id'] = str(member['_id'])
admin_user = User.find(organization.admin_uid, api=api)
# Make sure it's never None
organization.unknown_members = organization.unknown_members or []
can_super_edit = current_user.has_cap('admin')
can_edit = can_super_edit or om.user_is_admin(organization_oid)
csrf = flask_wtf.csrf.generate_csrf()
return render_template('organizations/view_embed.html',
organization=organization,
admin_user=admin_user,
members=members,
can_edit=can_edit,
can_super_edit=can_super_edit,
seats_used=len(members) +
len(organization.unknown_members),
csrf=csrf)
def patch_change_ownership(self, manager_id: bson.ObjectId, patch: dict):
"""Shares or un-shares the Manager with a user."""
man_man = current_flamenco.manager_manager
if not man_man.user_is_owner(mngr_doc_id=manager_id):
log.warning('User %s uses PATCH to (un)share manager %s, '
'but user is not owner of that Manager. Request denied.',
current_user_id(), manager_id)
raise wz_exceptions.Forbidden()
action = patch.get('action', '')
try:
action = man_man.ShareAction[action]
except KeyError:
raise wz_exceptions.BadRequest(f'Unknown action {action!r}')
subject_uid = str2id(patch.get('user', ''))
if action == man_man.ShareAction.share and subject_uid == current_user_id():
log.warning('%s tries to %s Manager %s with itself',
current_user_id(), action, manager_id)
raise wz_exceptions.BadRequest(f'Cannot share a Manager with yourself')
if action == man_man.ShareAction.share and \
not current_flamenco.auth.user_is_flamenco_user(subject_uid):
log.warning('%s Manager %s on behalf of user %s, but subject user %s '
'is not Flamenco user', action, manager_id, current_user_id(),
subject_uid)
raise wz_exceptions.Forbidden(f'User {subject_uid} is not allowed to use Flamenco')
try:
man_man.share_unshare_manager(manager_id, action, subject_uid)
except ValueError as ex:
raise wz_exceptions.BadRequest(str(ex))
def share_node(node_id):
"""Shares a node, or returns sharing information."""
node_id = str2id(node_id)
nodes_coll = current_app.data.driver.db['nodes']
node = nodes_coll.find_one({'_id': node_id},
projection={
'project': 1,
'node_type': 1,
'short_code': 1
})
if not node:
raise wz_exceptions.NotFound('Node %s does not exist.' % node_id)
check_permissions('nodes', node, request.method)
log.info('Sharing node %s', node_id)
short_code = node.get('short_code')
status = 200
if not short_code:
if request.method == 'POST':
short_code = generate_and_store_short_code(node)
make_world_gettable(node)
status = 201
else:
return '', 204
return jsonify(eve_hooks.short_link_info(short_code), status=status)
def patch(self, object_id: str):
from flask import request
import werkzeug.exceptions as wz_exceptions
from pillar.api.utils import str2id, authentication
# Parse the request
real_object_id = str2id(object_id)
patch = request.get_json()
if not patch:
self.log.info('Bad PATCH request, did not contain JSON')
raise wz_exceptions.BadRequest('Patch must contain JSON')
try:
patch_op = patch['op']
except KeyError:
self.log.info("Bad PATCH request, did not contain 'op' key")
raise wz_exceptions.BadRequest("PATCH should contain 'op' key to denote operation.")
log.debug('User %s wants to PATCH "%s" %s %s',
authentication.current_user_id(), patch_op, self.item_name, real_object_id)
# Find the PATCH handler for the operation.
try:
handler = self.patch_handlers[patch_op]
except KeyError:
log.warning('No %s PATCH handler for operation %r', self.item_name, patch_op)
raise wz_exceptions.BadRequest('Operation %r not supported' % patch_op)
# Let the PATCH handler do its thing.
response = handler(real_object_id, patch)
if response is None:
return '', 204
return response
def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job', 'Test job created from the server CLI', 'sleep', {
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
}, proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def patch_assign_user(self, org_id: bson.ObjectId, patch: dict):
"""Assigns a single user by User ID to an organization.
The calling user must be admin of the organization.
"""
from . import NotEnoughSeats
self._assert_is_admin(org_id)
# Do some basic validation.
try:
user_id = patch['user_id']
except KeyError:
raise wz_exceptions.BadRequest('No key "user_id" in patch.')
user_oid = str2id(user_id)
log.info('User %s uses PATCH to add user %s to organization %s',
current_user().user_id, user_oid, org_id)
try:
org_doc = current_app.org_manager.assign_single_user(
org_id, user_id=user_oid)
except NotEnoughSeats:
resp = jsonify(
{'_message': f'Not enough seats to assign this user'})
resp.status_code = 422
return resp
return jsonify(org_doc)
def get_community(project_id: str):
"""Fetch a valid community by its string id."""
# Cast to ObjectId
project_id = str2id(project_id)
# Ensure the project exists
projects_coll = current_app.db('projects')
community = projects_coll.find_one(
{
'_id': project_id,
'_deleted': {
'$ne': True
},
}, {
f'extension_props.{EXTENSION_NAME}': 1,
'name': 1
})
if not community:
log.debug('Project %s does not exist' % project_id)
return abort(404)
# Ensure the project is a community (was setup_for_dillo)
if EXTENSION_NAME not in community['extension_props']:
log.warning(
'Project %s is not a setup for Dillo and can not be followed' %
project_id)
return abort(403)
return community
def patch_node(node_id):
# Parse the request
node_id = str2id(node_id)
patch = request.get_json()
# Find the node type.
node = mongo.find_one_or_404('nodes', node_id, projection={'node_type': 1})
try:
node_type = node['node_type']
except KeyError:
msg = 'Node %s has no node_type property' % node_id
log.warning(msg)
raise wz_exceptions.InternalServerError(msg)
log.debug('User %s wants to PATCH %s node %s',
authentication.current_user_id(), node_type, node_id)
# Find the PATCH handler for the node type.
try:
patch_handler = custom.patch_handlers[node_type]
except KeyError:
log.info('No patch handler for node type %r', node_type)
raise wz_exceptions.MethodNotAllowed(
'PATCH on node type %r not allowed' % node_type)
# Let the PATCH handler do its thing.
return patch_handler(node_id, patch)
def revoke_auth_token(manager_id):
"""Revokes the Manager's existing authentication tokens.
Only allowed by owners of the Manager.
"""
manager_oid = str2id(manager_id)
csrf = request.form.get('csrf', '')
if not flask_wtf.csrf.validate_csrf(csrf):
log.warning(
'User %s tried to generate authentication token for Manager %s without '
'valid CSRF token!', current_user.user_id, manager_oid)
raise wz_exceptions.PreconditionFailed()
if not current_flamenco.manager_manager.user_is_owner(
mngr_doc_id=manager_oid):
log.warning(
'User %s wants to generate authentication token of manager %s, '
'but user is not owner of that Manager. Request denied.',
current_user.user_id, manager_oid)
raise wz_exceptions.Forbidden()
current_flamenco.manager_manager.revoke_auth_token(manager_oid)
return '', 204
def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url},
projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job',
'Test job created from the server CLI',
'sleep',
{
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
},
proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def create_manager_doc(self,
service_account_id,
name,
description,
url=None):
"""Creates a new Flamenco manager and its owner group.
Returns the MongoDB document.
"""
from pillar.api.utils import str2id
import bson
# Determine the Object IDs beforehand, so that the manager can refer to the
# group (by actual ID) and the group can mention the manager ID in the name.
manager_id = bson.ObjectId()
group_id = bson.ObjectId()
# Create an owner group for this manager.
group_doc = {
'_id': group_id,
'name': f'Owners of Flamenco Manager {manager_id}'
}
r, _, _, status = current_app.post_internal('groups', group_doc)
if status != 201:
self._log.error(
'Error creating manager owner group; status should be 201, not %i: %s',
status, r)
raise ValueError(
f'Unable to create Flamenco manager, status code {status}')
# Create the manager.
mngr_doc = {
'_id': manager_id,
'name': name,
'description': description,
'job_types': {
'sleep': {
'vars': {}
}
},
'service_account': str2id(service_account_id),
'owner': group_id,
}
if url:
mngr_doc['url'] = url
self._log.info('Creating manager %r at %s', name, url)
else:
self._log.info('Creating manager %r', name)
r, _, _, status = current_app.post_internal('flamenco_managers',
mngr_doc)
if status != 201:
self._log.error('Status should be 201, not %i: %s' % (status, r))
raise ValueError(
'Unable to create Flamenco manager, status code %i' % status)
mngr_doc.update(r)
return mngr_doc
def urler(project_id):
"""Returns the URL of any project."""
project_id = str2id(project_id)
project = mongo.find_one_or_404('projects',
project_id,
projection={'url': 1})
return jsonify({'_id': project_id, 'url': project['url']})
def assert_file_doc_ok(self, file_id, file_size):
with self.app.test_request_context():
from pillar.api.utils import str2id
# Check that the file exists in MongoDB
files_coll = self.app.data.driver.db['files']
db_file = files_coll.find_one({'_id': str2id(file_id)})
self.assertEqual(file_size, db_file['length'])
def project_timeline(pid_path: str):
continue_from_str = request.args.get('from')
continue_from = parse_continue_from(continue_from_str)
nbr_of_weeks_str = request.args.get('weeksToLoad')
nbr_of_weeks = parse_nbr_of_weeks(nbr_of_weeks_str)
sort_direction = request.args.get('dir', 'desc')
pid = str2id(pid_path)
return _project_timeline(continue_from, nbr_of_weeks, sort_direction, pid)
def recreate_job(project: pillarsdk.Project, job_id):
from pillar.api.utils.authentication import current_user_id
from pillar.api.utils import str2id
log.info('Recreating job %s on behalf of user %s', job_id, current_user_id())
job_id = str2id(job_id)
current_flamenco.api_recreate_job(job_id)
return '', 204
def recreate_job(project: pillarsdk.Project, job_id):
from pillar.api.utils.authentication import current_user_id
from pillar.api.utils import str2id
log.info('Recreating job %s on behalf of user %s', job_id, current_user_id())
job_id = str2id(job_id)
current_flamenco.api_recreate_job(job_id)
return '', 204
def setup_for_flamenco(project: pillarsdk.Project):
from pillar.api.utils import str2id
import flamenco.setup
project_id = project._id
if not project.has_method('PUT'):
log.warning('User %s tries to set up project %s for Flamenco, but has no PUT rights.',
current_user, project_id)
raise wz_exceptions.Forbidden()
if not current_flamenco.auth.current_user_is_flamenco_user():
log.warning('User %s tries to set up project %s for Flamenco, but is not flamenco-user.',
current_user, project_id)
raise wz_exceptions.Forbidden()
log.info('User %s sets up project %s for Flamenco', current_user, project_id)
flamenco.setup.setup_for_flamenco(project.url)
# Find the Managers available to this user, so we can auto-assign if there is exactly one.
man_man = current_flamenco.manager_manager
managers = man_man.owned_managers([bson.ObjectId(gid) for gid in current_user.groups])
manager_count = managers.count()
project_oid = str2id(project_id)
user_id = current_user_id()
if manager_count == 0:
_, mngr_doc, _ = man_man.create_new_manager('My Manager', '', user_id)
assign_man_oid = mngr_doc['_id']
log.info('Created and auto-assigning Manager %s to project %s upon setup for Flamenco.',
assign_man_oid, project_oid)
man_man.api_assign_to_project(assign_man_oid, project_oid, 'assign')
elif manager_count == 1:
assign_manager = managers.next()
assign_man_oid = str2id(assign_manager['_id'])
log.info('Auto-assigning Manager %s to project %s upon setup for Flamenco.',
assign_man_oid, project_oid)
man_man.api_assign_to_project(assign_man_oid, project_oid, 'assign')
return '', 204
def view_embed(manager_id: str):
if not request.is_xhr:
return index(manager_id)
api = pillar_api()
manager: Manager = Manager.find(manager_id, api=api)
linked_projects = manager.linked_projects(api=api)
linked_project_ids = set(manager.projects or [])
# TODO: support pagination
fetched = current_flamenco.flamenco_projects(projection={
'_id': 1,
'url': 1,
'name': 1,
})
available_projects = [
project for project in fetched._items
if project['_id'] not in linked_project_ids
]
owner_gid = str2id(manager['owner'])
owners = current_flamenco.manager_manager.owning_users(owner_gid)
for owner in owners:
owner['avatar'] = gravatar(owner.get('email'))
owner['_id'] = str(owner['_id'])
manager_oid = str2id(manager_id)
can_edit = current_flamenco.manager_manager.user_is_owner(
mngr_doc_id=manager_oid)
csrf = flask_wtf.csrf.generate_csrf()
return render_template('flamenco/managers/view_manager_embed.html',
manager=manager.to_dict(),
can_edit=can_edit,
available_projects=available_projects,
linked_projects=linked_projects,
owners=owners,
can_abandon_manager=len(owners) > 1,
csrf=csrf)
def sidebar_links(self, project):
from pillar.api.utils import str2id
if not self.is_flamenco_project(project):
return ''
project_id = str2id(project['_id'])
if not self.auth.current_user_may(self.auth.Actions.VIEW, project_id):
return ''
return flask.render_template('flamenco/sidebar.html', project=project)
def modify_access(self, project: pillarsdk.Project, repo_id: str, *,
grant_user_id: str = '', grant_passwd: str = '',
revoke_user_id: str = ''):
"""Grants or revokes access to/from the given user."""
if bool(grant_user_id) == bool(revoke_user_id):
raise ValueError('pass either grant_user_id or revoke_user_id, not both/none')
if grant_user_id:
grant_revoke = 'grant'
grant_passwd = self.hash_password(grant_passwd) if grant_passwd else UNSET_PASSWORD
else:
grant_revoke = 'revoke'
eprops, proj = self._get_prop_props(project)
proj_repo_id = eprops.get('repo_id')
proj_oid = str2id(proj['_id'])
if proj_repo_id != repo_id:
self._log.warning('project %s is linked to repo %r, not to %r, '
'refusing to %s access',
proj_oid, proj_repo_id, grant_revoke, repo_id)
raise ValueError()
users = eprops.setdefault('users', {})
if grant_user_id:
db_user = self._get_db_user(proj, repo_id, grant_user_id)
username = db_user['username']
grant = [(username, grant_passwd)]
revoke = []
users[grant_user_id] = {'username': username,
'pw_set': grant_passwd != UNSET_PASSWORD}
else:
user_info = users.pop(revoke_user_id, None)
if not user_info:
self._log.warning('unable to revoke user %s access from repo %s of project %s:'
' that user has no access', revoke_user_id, repo_id, proj_oid)
return
username = user_info['username']
grant = []
revoke = [username]
self._log.info('%sing user %s (%r) access to repo %s of project %s',
grant_revoke.rstrip('e'), grant_user_id or revoke_user_id, username,
repo_id, proj_oid)
self.remote.modify_access(repo_id, grant=grant, revoke=revoke)
proj_coll = current_app.db('projects')
res = proj_coll.update_one({'_id': proj_oid},
{'$set': {f'extension_props.{EXTENSION_NAME}.users': users}})
if res.matched_count != 1:
self._log.error('Matched count was %d, result: %s', res.matched_count, res.raw_result)
raise ValueError('Error updating MongoDB')
def sidebar_links(self, project):
from pillar.api.utils import str2id
if not self.is_flamenco_project(project):
return ''
project_id = str2id(project['_id'])
if not self.auth.current_user_may(self.auth.Actions.VIEW, project_id):
return ''
return flask.render_template('flamenco/sidebar.html', project=project)
def reset_token():
"""Generates a new authentication token for the Manager.
The Manager must have exchanged a secret key first, which must be linked to a Manager ID
before this function can be called.
"""
from flamenco import current_flamenco
from .linking_routes import check_hmac
data = request.get_json()
identifier = str2id(data.get('identifier'))
manager_id = str2id(data.get('manager_id'))
padding = data.get('padding', '')
mac = data.get('hmac')
log.info('Received request to reset auth token for Manager %s', manager_id)
mngr_key_coll = current_flamenco.db('manager_linking_keys')
key_info = mngr_key_coll.find_one({
'_id': identifier,
'manager_id': manager_id
})
if not key_info or not key_info.get('secret_key'):
log.warning('No secret key found for identifier %s, manager %s',
identifier, manager_id)
raise wz_exceptions.BadRequest('No secret key exchanged')
check_hmac(key_info['secret_key'],
f'{padding}-{identifier}-{manager_id}'.encode('ascii'), mac)
auth_token_info = current_flamenco.manager_manager.gen_new_auth_token(
manager_id)
if not auth_token_info:
raise wz_exceptions.NotFound()
del_res = mngr_key_coll.delete_many({'manager_id': manager_id})
log.info(
'Authentication token reset for Manager %s, all %d secret key(s) for this'
' manager have been removed.', manager_id, del_res.deleted_count)
return jsonify(attr.asdict(auth_token_info))
def move_group_node_project(node_uuid,
dest_proj_url,
force=False,
skip_gcs=False):
"""Copies all files from one project to the other, then moves the nodes.
The node and all its children are moved recursively.
"""
from pillar.api.nodes import moving
from pillar.api.utils import str2id
logging.getLogger('pillar').setLevel(logging.INFO)
db = current_app.db()
nodes_coll = db['nodes']
projs_coll = db['projects']
# Parse CLI args and get the node, source and destination projects.
node_uuid = str2id(node_uuid)
node = nodes_coll.find_one({'_id': node_uuid})
if node is None:
log.error("Node %s can't be found!", node_uuid)
return 1
if node.get('parent', None):
log.error('Node cannot have a parent, it must be top-level.')
return 4
src_proj = projs_coll.find_one({'_id': node['project']})
dest_proj = projs_coll.find_one({'url': dest_proj_url})
if src_proj is None:
log.warning("Node's source project %s doesn't exist!", node['project'])
if dest_proj is None:
log.error("Destination project url='%s' doesn't exist.", dest_proj_url)
return 2
if src_proj['_id'] == dest_proj['_id']:
if force:
log.warning("Node is already at project url='%s'!", dest_proj_url)
else:
log.error("Node is already at project url='%s'!", dest_proj_url)
return 3
log.info(
"Mass-moving %s (%s) and children from project '%s' (%s) to '%s' (%s)",
node_uuid, node['name'], src_proj['url'], src_proj['_id'],
dest_proj['url'], dest_proj['_id'])
mover = moving.NodeMover(db=db, skip_gcs=skip_gcs)
mover.change_project(node, dest_proj)
log.info('Done moving.')
def generate_token(manager_id: str):
manager_oid = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_oid)
# There are three ways in which a user can get here. One is authenticated via
# Authorization header (either Bearer token or Basic token:subtoken), and the
# other is via an already-existing browser session.
# In the latter case it's a redirect from a Flamenco Manager and we need to
# check the timeout and HMAC.
if not request.headers.get('Authorization'):
hasher = current_flamenco.manager_manager.hasher(manager_oid)
if hasher is None:
raise wz_exceptions.InternalServerError(
'Flamenco Manager not linked to this server')
expires = request.args.get('expires', '')
string_to_hash = f'{expires}-{manager_id}'
hasher.update(string_to_hash.encode('utf8'))
actual_hmac = hasher.hexdigest()
query_hmac = request.args.get('hmac', '')
if not hmac.compare_digest(query_hmac, actual_hmac):
raise wz_exceptions.Unauthorized('Bad HMAC')
# Only parse the timestamp after we learned we can trust it.
expire_timestamp = dateutil.parser.parse(expires)
validity_seconds_left = (expire_timestamp - utcnow()).total_seconds()
if validity_seconds_left < 0:
raise wz_exceptions.Unauthorized('Link expired')
if validity_seconds_left > 900:
# Flamenco Manager generates links that are valid for less than a minute, so
# if it's more than 15 minutes in the future, it's bad.
raise wz_exceptions.Unauthorized('Link too far in the future')
user = authentication.current_user()
if not current_flamenco.manager_manager.user_may_use(mngr_doc=manager):
log.warning(
'Account %s called %s for manager %s without access to that manager',
user.user_id, request.url, manager_oid)
raise wz_exceptions.Unauthorized()
jwt = current_flamenco.jwt
if not jwt.usable:
raise wz_exceptions.NotImplemented(
'JWT keystore is not usable at the moment')
log.info('Generating JWT key for user_id=%s manager_id=%s remote_addr=%s',
user.user_id, manager_id, request.remote_addr)
key_for_manager = jwt.generate_key_for_manager(manager_oid, user.user_id)
return Response(key_for_manager, content_type='text/plain')
def generate_token(manager_id: str):
manager_oid = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_oid)
# There are three ways in which a user can get here. One is authenticated via
# Authorization header (either Bearer token or Basic token:subtoken), and the
# other is via an already-existing browser session.
# In the latter case it's a redirect from a Flamenco Manager and we need to
# check the timeout and HMAC.
if not request.headers.get('Authorization'):
hasher = current_flamenco.manager_manager.hasher(manager_oid)
if hasher is None:
raise wz_exceptions.InternalServerError('Flamenco Manager not linked to this server')
expires = request.args.get('expires', '')
string_to_hash = f'{expires}-{manager_id}'
hasher.update(string_to_hash.encode('utf8'))
actual_hmac = hasher.hexdigest()
query_hmac = request.args.get('hmac', '')
if not hmac.compare_digest(query_hmac, actual_hmac):
raise wz_exceptions.Unauthorized('Bad HMAC')
# Only parse the timestamp after we learned we can trust it.
expire_timestamp = dateutil.parser.parse(expires)
validity_seconds_left = (expire_timestamp - utcnow()).total_seconds()
if validity_seconds_left < 0:
raise wz_exceptions.Unauthorized('Link expired')
if validity_seconds_left > 900:
# Flamenco Manager generates links that are valid for less than a minute, so
# if it's more than 15 minutes in the future, it's bad.
raise wz_exceptions.Unauthorized('Link too far in the future')
user = authentication.current_user()
if not current_flamenco.manager_manager.user_may_use(mngr_doc=manager):
log.warning(
'Account %s called %s for manager %s without access to that manager',
user.user_id, request.url, manager_oid)
raise wz_exceptions.Unauthorized()
jwt = current_flamenco.jwt
if not jwt.usable:
raise wz_exceptions.NotImplemented('JWT keystore is not usable at the moment')
log.info('Generating JWT key for user_id=%s manager_id=%s remote_addr=%s',
user.user_id, manager_id, request.remote_addr)
key_for_manager = jwt.generate_key_for_manager(manager_oid, user.user_id)
return Response(key_for_manager, content_type='text/plain')
def wrapper(manager_id, *args, **kwargs):
from flamenco import current_flamenco
from pillar.api.utils import str2id, mongo
manager_id = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_id)
if not current_flamenco.manager_manager.user_manages(mngr_doc=manager):
user_id = authentication.current_user_id()
log.warning(
'Service account %s sent startup notification for manager %s of another '
'service account', user_id, manager_id)
raise wz_exceptions.Unauthorized()
return wrapped(manager_id, request.json, *args, **kwargs)
def get_video_progress(video_id: str):
"""Return video progress information.
Either a `204 No Content` is returned (no information stored),
or a `200 Ok` with JSON from Eve's 'users' schema, from the key
video.view_progress.<video_id>.
"""
# Validation of the video ID; raises a BadRequest when it's not an ObjectID.
# This isn't strictly necessary, but it makes this function behave symmetrical
# to the set_video_progress() function.
utils.str2id(video_id)
users_coll = current_app.db('users')
user_doc = users_coll.find_one(current_user.user_id, projection={'nodes.view_progress': True})
try:
progress = user_doc['nodes']['view_progress'][video_id]
except KeyError:
return '', 204
if not progress:
return '', 204
return utils.jsonify(progress)
def assert_task_status(self, task_id, expected_status):
if isinstance(task_id, str):
from pillar.api.utils import str2id
task_id = str2id(task_id)
with self.app.test_request_context():
tasks_coll = self.flamenco.db('tasks')
task = tasks_coll.find_one({'_id': task_id})
self.assertIsNotNone(task, 'Task %s does not exist in the database' % task_id)
self.assertEqual(task['status'], str(expected_status),
"Task %s:\n has status: '%s'\n but expected: '%s'" % (
task_id, task['status'], expected_status))
return task
def attach_task_log(manager_id: ObjectId, _, task_id: str):
"""Store the POSTed task log as a file in the storage backend.
Also updates the task itself to have a reference to the file.
"""
# We only want to deal with GZipped files.
if 'logfile' not in request.files:
raise wz_exceptions.BadRequest("Missing uploaded file named 'logfile'")
uploaded_file: werkzeug.datastructures.FileStorage = request.files[
'logfile']
if not uploaded_file.filename.endswith('.gz'):
# The test HTTP client doesn't support setting per-part headers.
raise wz_exceptions.BadRequest(f'GZIP your file!')
# De-queue now; if the task or project doesn't exist, the Manager shouldn't be asked again.
task_oid = str2id(task_id)
current_flamenco.manager_manager.dequeue_task_log_request(
manager_id, task_oid)
# Check whether this Manager may attach to this Task.
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one({'_id': task_oid, 'manager': manager_id})
if not task:
raise wz_exceptions.NotFound(f'No such task exists')
proj_coll = current_app.db('projects')
project = proj_coll.find_one(
{
'_id': task['project'],
'_deleted': {
'$ne': True
}
},
projection={'url': True})
if not project:
log.warning('attach_task_log(%s, %s): project %s does not exist!',
manager_id, task_id, task['project'])
raise wz_exceptions.NotFound(
f'Project for task {task_oid} does not exist')
preexisting = current_flamenco.task_manager.api_attach_log(
task, uploaded_file)
resp = jsonify({'_message': 'ok'}, status=200 if preexisting else 201)
resp.headers['Location'] = url_for(
'flamenco.tasks.perproject.download_task_log_file',
project_url=project['url'],
task_id=task_id)
return resp
def get_allowed_methods(project_id=None, node_type=None):
"""Returns allowed methods to create a node of a certain type.
Either project_id or parent_node_id must be given. If the latter is given,
the former is deducted from it.
"""
project = mongo.find_one_or_404('projects', str2id(project_id))
proj_methods = authorization.compute_allowed_methods('projects', project, node_type)
resp = make_response()
resp.headers['Allowed'] = ', '.join(sorted(proj_methods))
resp.status_code = 204
return resp
def wrapper(manager_id, *args, **kwargs):
from pillar.api.utils import mongo
manager_id = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_id)
if not current_flamenco.manager_manager.user_manages(mngr_doc=manager):
user_id = authentication.current_user_id()
log.warning(
'Service account %s called manager API endpoint for manager %s of another '
'service account', user_id, manager_id)
raise wz_exceptions.Unauthorized()
return wrapped(manager if pass_manager_doc else manager_id,
request.json,
*args, **kwargs)
def reset_token():
"""Generates a new authentication token for the Manager.
The Manager must have exchanged a secret key first, which must be linked to a Manager ID
before this function can be called.
"""
from flamenco import current_flamenco
from .linking_routes import check_hmac
data = request.get_json()
identifier = str2id(data.get('identifier'))
manager_id = str2id(data.get('manager_id'))
padding = data.get('padding', '')
mac = data.get('hmac')
log.info('Received request to reset auth token for Manager %s', manager_id)
mngr_key_coll = current_flamenco.db('manager_linking_keys')
key_info = mngr_key_coll.find_one({'_id': identifier, 'manager_id': manager_id})
if not key_info or not key_info.get('secret_key'):
log.warning('No secret key found for identifier %s, manager %s', identifier, manager_id)
raise wz_exceptions.BadRequest('No secret key exchanged')
check_hmac(key_info['secret_key'],
f'{padding}-{identifier}-{manager_id}'.encode('ascii'),
mac)
auth_token_info = current_flamenco.manager_manager.gen_new_auth_token(manager_id)
if not auth_token_info:
raise wz_exceptions.NotFound()
del_res = mngr_key_coll.delete_many({'manager_id': manager_id})
log.info('Authentication token reset for Manager %s, all %d secret key(s) for this'
' manager have been removed.', manager_id, del_res.deleted_count)
return jsonify(attr.asdict(auth_token_info))
def _assign_or_remove_project(self, manager_id: bson.ObjectId, patch: dict,
action: str):
"""Assigns a manager to a project or removes it.
The calling user must be owner of the manager (always)
and member of the project (if assigning).
"""
from pillar.api.projects.utils import user_rights_in_project
from flamenco import current_flamenco
try:
project_strid = patch['project']
except KeyError:
log.warning('User %s sent invalid PATCH %r for manager %s.',
current_user_id(), patch, manager_id)
raise wz_exceptions.BadRequest('Missing key "project"')
project_id = str2id(project_strid)
if not current_flamenco.manager_manager.user_is_owner(
mngr_doc_id=manager_id):
log.warning(
'User %s uses PATCH to %s manager %s to/from project %s, '
'but user is not owner of that Manager. Request denied.',
current_user_id(), action, manager_id, project_id)
raise wz_exceptions.Forbidden()
# Removing from a project doesn't require project membership.
if action != 'remove':
methods = user_rights_in_project(project_id)
if 'PUT' not in methods:
log.warning(
'User %s uses PATCH to %s manager %s to/from project %s, '
'but only has %s rights on project. Request denied.',
current_user_id(), action, manager_id, project_id,
', '.join(methods))
raise wz_exceptions.Forbidden()
log.info('User %s uses PATCH to %s manager %s to/from project %s',
current_user_id(), action, manager_id, project_id)
ok = current_flamenco.manager_manager.api_assign_to_project(
manager_id, project_id, action)
if not ok:
# Manager Manager will have already logged the cause.
raise wz_exceptions.InternalServerError()
def org_members(self, member_sting_ids: typing.Iterable[str]) -> typing.List[dict]:
"""Returns the user documents of the organization members.
This is a workaround to provide membership information for
organizations without giving 'mortal' users access to /api/users.
"""
from pillar.api.utils import str2id
if not member_sting_ids:
return []
member_ids = [str2id(uid) for uid in member_sting_ids]
users_coll = current_app.db('users')
users = users_coll.find({'_id': {'$in': member_ids}},
projection={'_id': 1, 'full_name': 1, 'email': 1})
return list(users)
def project_settings(project: pillarsdk.Project, **template_args: dict):
"""Renders the project settings page for Flamenco projects."""
from pillar.api.utils import str2id
from pillar.web.system_util import pillar_api
from .managers.sdk import Manager
if not current_flamenco.auth.current_user_is_flamenco_user():
raise wz_exceptions.Forbidden()
# Based on the project state, we can render a different template.
if not current_flamenco.is_flamenco_project(project):
return render_template('flamenco/project_settings/offer_setup.html',
project=project,
**template_args)
project_id = str2id(project['_id'])
flauth = current_flamenco.auth
may_use = flauth.current_user_may(flauth.Actions.USE, project_id)
# Use the API for querying for Managers, because it implements access control.
api = pillar_api()
managers = Manager.all(api=api)
linked_managers = Manager.all({
'where': {
'projects': project['_id'],
},
},
api=api)
try:
first_manager = managers['_items'][0]
except (KeyError, IndexError):
first_manager = None
try:
first_linked_manager = linked_managers['_items'][0]
except (KeyError, IndexError):
first_linked_manager = None
return render_template('flamenco/project_settings/settings.html',
project=project,
managers=managers,
first_manager=first_manager,
linked_managers=linked_managers,
first_linked_manager=first_linked_manager,
may_use_flamenco=may_use,
**template_args)
def project_settings(self, project: pillarsdk.Project, **template_args: dict) -> flask.Response:
"""Renders the project settings page for this extension.
Set YourExtension.has_project_settings = True and Pillar will call this function.
:param project: the project for which to render the settings.
:param template_args: additional template arguments.
:returns: a Flask HTTP response
"""
from .routes import project_settings
if not self.is_svnman_project(project):
return flask.render_template('svnman/project_settings/offer_create_repo.html',
project=project, **template_args)
remote_url = current_app.config['SVNMAN_REPO_URL']
users_coll = current_app.db('users')
# list of {'username': '******', 'db': user in our DB, 'pw_is_set': bool} dicts.
svn_users = []
eprops = project.extension_props[EXTENSION_NAME]
repo_id = eprops.repo_id
svn_url = urljoin(remote_url, repo_id)
if eprops.users: # may be None
userdict = eprops.users.to_dict()
# Jump through some hoops to collect the user info from MongoDB in one query.
svninfo = {str2id(uid): userinfo for uid, userinfo in userdict.items()}
db_users = users_coll.find(
{'_id': {'$in': list(svninfo.keys())}},
projection={'full_name': 1, 'email': 1, 'avatar': 1},
)
for db_user in db_users:
svninfo.setdefault(db_user['_id'], {})['db'] = db_user
db_user['avatar_url'] = pillar.api.users.avatar.url(db_user)
svn_users = sorted(svninfo.values(), key=lambda item: item.get('username', ''))
return flask.render_template('svnman/project_settings/settings.html',
project=project,
svn_url=svn_url,
repo_id=repo_id,
remote_url=remote_url,
svn_users=svn_users,
**template_args)
def project_settings(project: pillarsdk.Project, **template_args: dict):
"""Renders the project settings page for Flamenco projects."""
from pillar.api.utils import str2id
from pillar.web.system_util import pillar_api
from .managers.sdk import Manager
if not current_flamenco.auth.current_user_is_flamenco_user():
raise wz_exceptions.Forbidden()
# Based on the project state, we can render a different template.
if not current_flamenco.is_flamenco_project(project):
return render_template('flamenco/project_settings/offer_setup.html',
project=project, **template_args)
project_id = str2id(project['_id'])
flauth = current_flamenco.auth
may_use = flauth.current_user_may(flauth.Actions.USE, project_id)
# Use the API for querying for Managers, because it implements access control.
api = pillar_api()
managers = Manager.all(api=api)
linked_managers = Manager.all({
'where': {
'projects': project['_id'],
},
}, api=api)
try:
first_manager = managers['_items'][0]
except (KeyError, IndexError):
first_manager = None
try:
first_linked_manager = linked_managers['_items'][0]
except (KeyError, IndexError):
first_linked_manager = None
return render_template('flamenco/project_settings/settings.html',
project=project,
managers=managers,
first_manager=first_manager,
linked_managers=linked_managers,
first_linked_manager=first_linked_manager,
may_use_flamenco=may_use,
**template_args)
def _manager_project(manager_id, project_url, action):
from pillar.api.utils import str2id
from flamenco import current_flamenco
authentication.force_cli_user()
manager_id = str2id(manager_id)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
project_id = proj['_id']
ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action)
if not ok:
log.error('Unable to assign manager %s to project %s', manager_id, project_id)
return 1
def _assign_or_remove_project(self, manager_id: bson.ObjectId, patch: dict, action: str):
"""Assigns a manager to a project or removes it.
The calling user must be owner of the manager (always)
and member of the project (if assigning).
"""
from pillar.api.projects.utils import user_rights_in_project
from flamenco import current_flamenco
try:
project_strid = patch['project']
except KeyError:
log.warning('User %s sent invalid PATCH %r for manager %s.',
current_user_id(), patch, manager_id)
raise wz_exceptions.BadRequest('Missing key "project"')
project_id = str2id(project_strid)
if not current_flamenco.manager_manager.user_is_owner(mngr_doc_id=manager_id):
log.warning('User %s uses PATCH to %s manager %s to/from project %s, '
'but user is not owner of that Manager. Request denied.',
current_user_id(), action, manager_id, project_id)
raise wz_exceptions.Forbidden()
# Removing from a project doesn't require project membership.
if action != 'remove':
methods = user_rights_in_project(project_id)
if 'PUT' not in methods:
log.warning('User %s uses PATCH to %s manager %s to/from project %s, '
'but only has %s rights on project. Request denied.',
current_user_id(), action, manager_id, project_id, ', '.join(methods))
raise wz_exceptions.Forbidden()
log.info('User %s uses PATCH to %s manager %s to/from project %s',
current_user_id(), action, manager_id, project_id)
ok = current_flamenco.manager_manager.api_assign_to_project(
manager_id, project_id, action)
if not ok:
# Manager Manager will have already logged the cause.
raise wz_exceptions.InternalServerError()
def attach_task_log(manager_id: ObjectId, _, task_id: str):
"""Store the POSTed task log as a file in the storage backend.
Also updates the task itself to have a reference to the file.
"""
# We only want to deal with GZipped files.
if 'logfile' not in request.files:
raise wz_exceptions.BadRequest("Missing uploaded file named 'logfile'")
uploaded_file: werkzeug.datastructures.FileStorage = request.files['logfile']
if not uploaded_file.filename.endswith('.gz'):
# The test HTTP client doesn't support setting per-part headers.
raise wz_exceptions.BadRequest(f'GZIP your file!')
# De-queue now; if the task or project doesn't exist, the Manager shouldn't be asked again.
task_oid = str2id(task_id)
current_flamenco.manager_manager.dequeue_task_log_request(manager_id, task_oid)
# Check whether this Manager may attach to this Task.
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one({'_id': task_oid, 'manager': manager_id})
if not task:
raise wz_exceptions.NotFound(f'No such task exists')
proj_coll = current_app.db('projects')
project = proj_coll.find_one({'_id': task['project'], '_deleted': {'$ne': True}},
projection={'url': True})
if not project:
log.warning('attach_task_log(%s, %s): project %s does not exist!',
manager_id, task_id, task['project'])
raise wz_exceptions.NotFound(f'Project for task {task_oid} does not exist')
preexisting = current_flamenco.task_manager.api_attach_log(task, uploaded_file)
resp = jsonify({'_message': 'ok'}, status=200 if preexisting else 201)
resp.headers['Location'] = url_for(
'flamenco.tasks.perproject.download_task_log_file',
project_url=project['url'], task_id=task_id)
return resp
def index():
user_id = current_user.user_id
# Fetch available Managers.
man_man = current_flamenco.manager_manager
managers = list(man_man.owned_managers(
current_user.group_ids, {'_id': 1, 'name': 1}))
manager_limit_reached = not current_user.has_cap('admin') and \
len(managers) >= flamenco.auth.MAX_MANAGERS_PER_USER
# Get the query arguments
identifier: str = request.args.get('identifier', '')
return_url: str = request.args.get('return', '')
request_hmac: str = request.args.get('hmac', '')
ident_oid = str2id(identifier)
keys_coll = current_flamenco.db('manager_linking_keys')
# Verify the received identifier and return URL.
key_info = keys_coll.find_one({'_id': ident_oid})
if key_info is None:
log.warning('User %s tries to link a manager with an identifier that cannot be found',
user_id)
return render_template('flamenco/managers/linking/key_not_found.html')
check_hmac(key_info['secret_key'],
f'{identifier}-{return_url}'.encode('ascii'),
request_hmac)
# Only deal with POST data after HMAC verification is performed.
if request.method == 'POST':
manager_id = request.form['manager-id']
if manager_id == 'new':
manager_name = request.form.get('manager-name', '')
if not manager_name:
raise wz_exceptions.UnprocessableEntity('no Manager name given')
if manager_limit_reached:
log.warning('User %s tries to create a manager %r, but their limit is reached.',
user_id, manager_name)
raise wz_exceptions.UnprocessableEntity('Manager count limit reached')
log.info('Creating new Manager named %r for user %s', manager_name, user_id)
account, mngr_doc, token_data = man_man.create_new_manager(manager_name, '', user_id)
manager_oid = mngr_doc['_id']
else:
log.info('Linking existing Manager %r for user %s', manager_id, user_id)
manager_oid = str2id(manager_id)
# Store that this identifier belongs to this ManagerID
update_res = keys_coll.update_one({'_id': ident_oid},
{'$set': {'manager_id': manager_oid}})
if update_res.matched_count != 1:
log.error('Match count was %s when trying to update secret key '
'for Manager %s on behalf of user %s',
update_res.matched_count, manager_oid, user_id)
raise wz_exceptions.InternalServerError('Unable to store manager ID')
log.info('Manager ID %s is stored as belonging to key with identifier %s',
manager_oid, identifier)
# Now redirect the user back to Flamenco Manager in a verifyable way.
msg = f'{identifier}-{manager_oid}'.encode('ascii')
mac = _compute_hash(key_info['secret_key'], msg)
qs = urllib.parse.urlencode({
'hmac': mac,
'oid': str(manager_oid),
})
direct_to = urllib.parse.urljoin(return_url, f'?{qs}')
log.info('Directing user to URL %s', direct_to)
return redirect(direct_to, 307)
return render_template('flamenco/managers/linking/choose_manager.html',
managers=managers,
can_create_manager=not manager_limit_reached)
def handle_task_update_batch(manager_id, task_updates):
"""Performs task updates.
Task status changes are generally always accepted. The only exception is when the
task ID is contained in 'tasks_to_cancel'; in that case only a transition to either
'canceled', 'completed' or 'failed' is accepted.
:returns: tuple (total nr of modified tasks, handled update IDs)
"""
if not task_updates:
return 0, []
import dateutil.parser
from pillar.api.utils import str2id
from flamenco import current_flamenco, eve_settings
log.debug('Received %i task updates from manager %s', len(task_updates), manager_id)
tasks_coll = current_flamenco.db('tasks')
logs_coll = current_flamenco.db('task_logs')
valid_statuses = set(eve_settings.tasks_schema['status']['allowed'])
handled_update_ids = []
total_modif_count = 0
for task_update in task_updates:
# Check that this task actually belongs to this manager, before we accept any updates.
update_id = str2id(task_update['_id'])
task_id = str2id(task_update['task_id'])
task_info = tasks_coll.find_one({'_id': task_id},
projection={'manager': 1, 'status': 1, 'job': 1})
# For now, we just ignore updates to non-existing tasks. Someone might have just deleted
# one, for example. This is not a reason to reject the entire batch.
if task_info is None:
log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring',
manager_id, task_id)
handled_update_ids.append(update_id)
continue
if task_info['manager'] != manager_id:
log.warning('Manager %s sent update for task %s which belongs to other manager %s',
manager_id, task_id, task_info['manager'])
continue
if task_update.get('received_on_manager'):
received_on_manager = dateutil.parser.parse(task_update['received_on_manager'])
else:
# Fake a 'received on manager' field; it really should have been in the JSON payload.
received_on_manager = utcnow()
# Store the log for this task, allowing for duplicate log reports.
#
# NOTE: is deprecated and will be removed in a future version of Flamenco;
# only periodically send the last few lines of logging in 'log_tail' and
# store the entire log on the Manager itself.
task_log = task_update.get('log')
if task_log:
log_doc = {
'_id': update_id,
'task': task_id,
'received_on_manager': received_on_manager,
'log': task_log
}
logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True)
# Modify the task, and append the log to the logs collection.
updates = {
'task_progress_percentage': task_update.get('task_progress_percentage', 0),
'current_command_index': task_update.get('current_command_index', 0),
'command_progress_percentage': task_update.get('command_progress_percentage', 0),
'_updated': received_on_manager,
'_etag': random_etag(),
}
new_status = determine_new_task_status(manager_id, task_id, task_info,
task_update.get('task_status'), valid_statuses)
if new_status:
updates['status'] = new_status
new_activity = task_update.get('activity')
if new_activity:
updates['activity'] = new_activity
worker = task_update.get('worker')
if worker:
updates['worker'] = worker
metrics_timing = task_update.get('metrics', {}).get('timing')
if metrics_timing:
updates['metrics.timing'] = metrics_timing
fbw = task_update.get('failed_by_workers')
if fbw is not None:
updates['failed_by_workers'] = fbw
# Store the last lines of logging on the task itself.
task_log_tail: str = task_update.get('log_tail')
if not task_log_tail and task_log:
task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:])
if task_log_tail:
updates['log'] = task_log_tail
result = tasks_coll.update_one({'_id': task_id}, {'$set': updates})
total_modif_count += result.modified_count
handled_update_ids.append(update_id)
# Update the task's job after updating the task itself.
if new_status:
current_flamenco.job_manager.update_job_after_task_status_change(
task_info['job'], task_id, new_status)
return total_modif_count, handled_update_ids
