def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job', 'Test job created from the server CLI', 'sleep', {
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
}, proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def process_posts(community_name):
"""Manually trigger pre-update hooks."""
from flask import g
from pillar.auth import UserClass
from dillo.api.posts.hooks import process_picture_oembed, before_replacing_post
from dillo.setup import _get_project
project = _get_project(community_name)
nodes_collection = current_app.db()['nodes']
user_collection = current_app.db()['users']
nc = nodes_collection.find({
'node_type': 'dillo_post',
'properties.status': 'published',
'project': project['_id'],
})
for n in nc:
# Log in as doc user
user_doc = user_collection.find_one({'_id': n['user']})
u = UserClass.construct(user_doc['_id'], user_doc)
g.current_user = u
n_id = n['_id']
print(f'Processing node {n_id}')
process_picture_oembed(n, n)
before_replacing_post(n, n)
nodes_collection.find_one_and_replace({'_id': n_id}, n)
def create_test_job(manager_id, user_email, project_url):
"""Creates a test job for a given manager."""
from pillar.api.utils import dumps, str2id
manager_id = str2id(manager_id)
authentication.force_cli_user()
# Find user
users_coll = current_app.db()['users']
user = users_coll.find_one({'email': user_email}, projection={'_id': 1})
if not user:
raise ValueError('User with email %r not found' % user_email)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url},
projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
# Create the job
job = flamenco.current_flamenco.job_manager.api_create_job(
'CLI test job',
'Test job created from the server CLI',
'sleep',
{
'frames': '1-30, 40-44',
'chunk_size': 13,
'time_in_seconds': 3,
},
proj['_id'], user['_id'], manager_id)
log.info('Job created:\n%s', dumps(job, indent=4))
def select(self, sql, campos=None, sel='fetchone'):
"""Seleciona Registros da Tabela."""
if sel == 'fetchone':
model = None
items = current_app.db().select(sql, campos, sel)
if items:
model = self.__class__(items)
elif sel == 'fetchall':
model = current_app.db().select(sql, campos, sel)
model = [self.__class__(i) for i in model]
return model
def patch_post(node_id, patch):
assert_is_valid_patch(node_id, patch)
user_id = authentication.current_user_id()
if patch['op'] in COMMENT_VOTING_OPS:
nodes_coll = current_app.db()['nodes']
node = nodes_coll.find_one({'_id': node_id})
old_rating = rating_difference(node)
result, node = vote_comment(user_id, node_id, patch)
new_rating = rating_difference(node)
# Update the user karma based on the rating differences.
karma_increase = (new_rating - old_rating) * POST_VOTE_WEIGHT
if karma_increase != 0:
node_user_id = nodes_coll.find_one({'_id': node_id},
projection={
'user': 1,
})['user']
users_collection = current_app.db()['users']
db_fieldname = f'extension_props_public.{EXTENSION_NAME}.karma'
users_collection.find_one_and_update(
{'_id': node_user_id},
{'$inc': {
db_fieldname: karma_increase
}},
{db_fieldname: 1},
)
# Fetch the full node for updating hotness and reindexing
# TODO (can be improved by making a partial update)
node = nodes_coll.find_one({'_id': node['_id']})
update_hot(node)
nodes_coll.update_one(
{'_id': node['_id']},
{'$set': {
'properties.hot': node['properties']['hot']
}})
algolia_index_post_save(node)
else:
return abort(403)
return jsonify({
'_status': 'OK',
'result': result,
'properties': node['properties']
})
def update_download_count(post_id):
"""Update download count for the Post.
This function is called from dillo.web.posts.routes.download_file.
"""
current_app.db('nodes').update_one({'_id': ObjectId(post_id)}, {
'$inc': {
'properties.downloads_total': 1,
'properties.downloads_latest': 1
}
})
log.debug('Updated download count for post %s' % post_id)
return jsonify({'status': 'OK'})
def unfollow(project_id: str):
"""Remove a community from followed_communities of current_user."""
# Ensure that the community requested exists
community = get_community(project_id)
users_coll = current_app.db('users')
# Fetch user
user = users_coll.find_one(current_user.user_id)
# Look for project in in extension_props_public.dillo.followed_communities
followed_communities = user['extension_props_public'][EXTENSION_NAME]. \
get('followed_communities', [])
# Check if the user already does not follow the community
if followed_communities and community['_id'] not in followed_communities:
log.debug('User does not follow community %s' % community['url'])
return abort(403)
followed_communities = [
c for c in followed_communities if c != community['_id']
]
followed_communities_key = f'extension_props_public.{EXTENSION_NAME}.followed_communities'
users_coll.update_one(
{'_id': current_user.user_id},
{'$set': {
followed_communities_key: followed_communities
}})
log.debug('Community %s successfully unfollowed' % community['name'])
return jsonify({
'_status': 'OK',
'message': f"Unfollowed {community['name']}"
})
def get_file_url(file_id: ObjectId, variation='') -> str:
"""Return the URL of a file in storage.
Note that this function is cached, see setup_app().
:param file_id: the ID of the file
:param variation: if non-empty, indicates the variation of of the file
to return the URL for; if empty, returns the URL of the original.
:return: the URL, or an empty string if the file/variation does not exist.
"""
file_coll = current_app.db('files')
db_file = file_coll.find_one({'_id': file_id})
if not db_file:
return ''
ensure_valid_link(db_file)
if variation:
variations = file_doc.get('variations', ())
for file_var in variations:
if file_var['size'] == variation:
return file_var['link']
return ''
return db_file['link']
def create_manager(owner_email, name, description):
"""Creates a Flamenco manager with service account.
:returns: tuple (mngr_doc, account, token)
"""
from pymongo.cursor import Cursor
# Find the owner, to add it to the owner group afterward.
possible_owners: Cursor = current_app.db('users').find(
{'email': owner_email}, {
'_id': 1,
'full_name': 1
})
owner_count = possible_owners.count()
if owner_count == 0:
raise ValueError(f'No user found with email address {owner_email}; '
'cannot assign ownership of Manager')
if owner_count > 1:
raise ValueError(
f'Multiple users ({owner_count}) found with email address {owner_email}; '
'cannot assign ownership of Manager')
owner = possible_owners[0]
owner_id = owner['_id']
account, mngr_doc, token = current_flamenco.manager_manager.create_new_manager(
name, description, owner_id)
return mngr_doc, account, token
def create_manager(owner_email, name, description):
"""Creates a Flamenco manager with service account.
:returns: tuple (mngr_doc, account, token)
"""
from pymongo.cursor import Cursor
# Find the owner, to add it to the owner group afterward.
possible_owners: Cursor = current_app.db('users').find(
{'email': owner_email},
{'_id': 1, 'full_name': 1})
owner_count = possible_owners.count()
if owner_count == 0:
raise ValueError(f'No user found with email address {owner_email}; '
'cannot assign ownership of Manager')
if owner_count > 1:
raise ValueError(f'Multiple users ({owner_count}) found with email address {owner_email}; '
'cannot assign ownership of Manager')
owner = possible_owners[0]
owner_id = owner['_id']
account, mngr_doc, token = current_flamenco.manager_manager.create_new_manager(
name, description, owner_id)
return mngr_doc, account, token
def generate_shortcode(project_id, node_type):
"""Generates and returns a new shortcode.
:param project_id: project ID
:type project_id: bson.ObjectId
:param node_type: the type, for now 'dillo_post', but can be extended.
:type node_type: unicode
"""
assert isinstance(project_id, ObjectId)
db = current_app.db()
db_fieldname = 'extension_props.dillo.last_used_shortcodes.%s' % node_type
log.debug('Incrementing project %s shortcode for type %r',
project_id, node_type)
new_proj = db['projects'].find_one_and_update(
{'_id': project_id},
{'$inc': {db_fieldname: 1}},
{db_fieldname: 1},
return_document=pymongo.ReturnDocument.AFTER,
)
shortcode = new_proj['extension_props']['dillo']['last_used_shortcodes'][node_type]
return encode_id(shortcode)
def rebuild_karma():
"""Re-calculate users karma
It also initialize the ['extension_props_public']['karma'] if needed.
"""
db = current_app.db()
users_collection = db['users'].find({'_deleted': {'$ne': True}})
db_fieldname = f'extension_props_public.{EXTENSION_NAME}.karma'
for user in users_collection:
posts_collection = db['nodes'].find({
'node_type': 'dillo_post',
'user': user['_id'],
})
karma = 0
for post in posts_collection:
karma += rating_difference(post) * POST_VOTE_WEIGHT
db['users'].find_one_and_update(
{'_id': user['_id']},
{'$set': {
db_fieldname: karma
}},
{db_fieldname: 1},
)
def assign_to_user_main(user_doc: dict):
"""Make every user create part of the user_main group."""
groups_coll = current_app.db().groups
group_dillo_user_main = groups_coll.find_one({'name': 'dillo_user_main'})
add_user_to_group(user_id=user_doc['_id'],
group_id=group_dillo_user_main['_id'])
def follow(project_id: str):
"""Add the community to followed_communities."""
# Ensure that the community requested exists
community = get_community(project_id)
users_coll = current_app.db('users')
# Fetch user
user = users_coll.find_one(current_user.user_id)
# Look for project in in extension_props_public.dillo.followed_communities
followed_communities = user['extension_props_public'][EXTENSION_NAME].\
get('followed_communities', [])
# Check if the user already follows the community
if followed_communities and community['_id'] in followed_communities:
log.debug('User already follows community %s' % community['url'])
return abort(403)
# Append the followed community to the existing list
followed_communities.append(community['_id'])
followed_communities_key = f'extension_props_public.{EXTENSION_NAME}.followed_communities'
users_coll.update_one(
{'_id': current_user.user_id},
{'$set': {
followed_communities_key: followed_communities
}})
return jsonify({
'_status': 'OK',
'message': f"Following {community['name']}"
})
def update(self, campo, val):
"""Atualiza os campos da tabela."""
dic = self.getDict()
campos = ', '.join(
map(lambda x: ' = '.join(x),
zip(
filter(
lambda x: not (self.__pk__ == x),
dic.keys()
),
map(lambda x: f"%({x})s",
filter(
lambda x: not (self.__pk__ == x),
dic.keys()
)
)
)
)
)
dic[campo] = val
return current_app.db().operacao(
f"UPDATE {self.__table__} "
f"SET {campos} where {campo} = %({campo})s",
dic
)
def get_community(project_id: str):
"""Fetch a valid community by its string id."""
# Cast to ObjectId
project_id = str2id(project_id)
# Ensure the project exists
projects_coll = current_app.db('projects')
community = projects_coll.find_one(
{
'_id': project_id,
'_deleted': {
'$ne': True
},
}, {
f'extension_props.{EXTENSION_NAME}': 1,
'name': 1
})
if not community:
log.debug('Project %s does not exist' % project_id)
return abort(404)
# Ensure the project is a community (was setup_for_dillo)
if EXTENSION_NAME not in community['extension_props']:
log.warning(
'Project %s is not a setup for Dillo and can not be followed' %
project_id)
return abort(403)
return community
def _tagged(tag: str):
"""Fetch all public nodes with the given tag.
This function is cached, see setup_app().
"""
nodes_coll = current_app.db('nodes')
agg = nodes_coll.aggregate([
{'$match': {'properties.tags': tag,
'_deleted': {'$ne': True}}},
# Only get nodes from public projects. This is done after matching the
# tagged nodes, because most likely nobody else will be able to tag
# nodes anyway.
{'$lookup': {
'from': 'projects',
'localField': 'project',
'foreignField': '_id',
'as': '_project',
}},
{'$unwind': '$_project'},
{'$match': {'_project.is_private': False}},
{'$addFields': {
'project._id': '$_project._id',
'project.name': '$_project.name',
'project.url': '$_project.url',
}},
# Don't return the entire project/file for each node.
{'$project': {'_project': False}},
{'$sort': {'_created': -1}}
])
return list(agg)
def add_communities_filter(pipeline):
"""Given an aggregation pipeline, add a filter for communities."""
current_user = pillar.auth.get_current_user()
default_followed_community_ids = current_app.config[
'DEFAULT_FOLLOWED_COMMUNITY_IDS']
# Add filter for communities
if current_user.is_anonymous and default_followed_community_ids:
pipeline[0]['$match']['project'] = {
'$in': default_followed_community_ids
}
elif current_user.is_authenticated:
users_coll = current_app.db('users')
followed_communities_key = f'extension_props_public.{EXTENSION_NAME}.followed_communities'
# TODO(fsiddi) project into a shorter key instead of followed_communities
followed = users_coll.find_one(
{
'_id': current_user.user_id,
followed_communities_key: {
'$exists': True
}
}, {followed_communities_key: 1})
# If current user is following communities, use them as filter
if followed:
fcl = followed['extension_props_public'][EXTENSION_NAME][
'followed_communities']
communities = [c for c in fcl]
pipeline[0]['$match']['project'] = {'$in': communities}
# Otherwise, try to use default communities
elif default_followed_community_ids:
pipeline[0]['$match']['project'] = {
'$in': default_followed_community_ids
}
def user_group_action(user_id: bson.ObjectId, group_id: bson.ObjectId,
action: str):
"""Performs a group action (add/remove).
:param user_id: the user's ObjectID.
:param group_id: the group's ObjectID.
:param action: either '$pull' to remove from a group, or '$addToSet' to add to a group.
"""
from pymongo.results import UpdateResult
assert isinstance(user_id, bson.ObjectId)
assert isinstance(group_id, bson.ObjectId)
assert action in {'$pull', '$addToSet'}
users_coll = current_app.db('users')
result: UpdateResult = users_coll.update_one(
{'_id': user_id},
{action: {
'groups': group_id
}},
)
if result.matched_count == 0:
raise ValueError(
f'Unable to {action} user {user_id} membership of group {group_id}; '
f'user not found.')
def insert(self):
"""Insere os dados na tabela com os campos adicionados."""
campos = ", ".join(self.getDict().keys())
values = ", ".join(map(lambda x: f"%({x})s", self.getDict().keys()))
return current_app.db().operacao(
f"INSERT INTO {self.__table__}({campos}) VALUES ({values})",
self.getDict()
)
def _public_project_ids() -> typing.List[bson.ObjectId]:
"""Returns a list of ObjectIDs of public projects.
Memoized in setup_app().
"""
proj_coll = current_app.db('projects')
result = proj_coll.find({'is_private': False}, {'_id': 1})
return [p['_id'] for p in result]
def remove_token(token: str):
"""Removes the token from the database."""
tokens_coll = current_app.db('tokens')
token_hashed = hash_auth_token(token)
# TODO: remove matching on hashed tokens once all hashed tokens have expired.
lookup = {'$or': [{'token': token}, {'token_hashed': token_hashed}]}
del_res = tokens_coll.delete_many(lookup)
log.debug('Removed token %r, matched %d documents', token,
del_res.deleted_count)
def user_rights_in_project(project_id: ObjectId) -> frozenset:
"""Returns the set of HTTP methods allowed on the given project for the current user."""
from pillar.api.utils import authorization
assert isinstance(project_id, ObjectId)
proj_coll = current_app.db().projects
proj = proj_coll.find_one({'_id': project_id})
return frozenset(authorization.compute_allowed_methods('projects', proj))
def _parent_name(task):
"""Returns 'for shot "shotname"' if the task is attached to the shot."""
parent = task.get('parent')
if not parent:
return ''
nodes_coll = current_app.db()['nodes']
shot = nodes_coll.find_one(parent)
if shot:
return ' for shot "%s"' % shot['name']
return ''
def notification_get_subscriptions(context_object_type, context_object_id,
actor_user_id):
subscriptions_collection = current_app.db('activities-subscriptions')
lookup = {
'user': {
"$ne": actor_user_id
},
'context_object_type': context_object_type,
'context_object': context_object_id,
'is_subscribed': True,
}
return subscriptions_collection.find(
lookup), subscriptions_collection.count_documents(lookup)
def count_blender_sync(query=None) -> int:
pipeline = [
# 0 Find all startups.blend that are not deleted
{
'$match': {
'_deleted': {
'$ne': 'true'
},
'name': 'startup.blend',
}
},
# 1 Group them per project (drops any duplicates)
{
'$group': {
'_id': '$project'
}
},
# 2 Join the project info
{
'$lookup': {
'from': "projects",
'localField': "_id",
'foreignField': "_id",
'as': "project",
}
},
# 3 Unwind the project list (there is always only one project)
{
'$unwind': {
'path': '$project',
}
},
# 4 Find all home projects
{
'$match': {
'project.category': 'home'
}
},
{
'$count': 'tot'
}
]
c = current_app.db()['nodes']
# If we provide a query, we extend the first $match step in the aggregation pipeline with
# with the extra parameters (for example _created)
if query:
pipeline[0]['$match'].update(query)
# Return either a list with one item or an empty list
r = list(c.aggregate(pipeline=pipeline))
count = 0 if not r else r[0]['tot']
return count
def get_user_list(user_list):
if not user_list:
return u'-nobody-'
user_coll = current_app.db()['users']
users = user_coll.find({'_id': {
'$in': user_list
}},
projection={
'full_name': 1,
})
names = [user['full_name'] for user in users]
return u', '.join(names)
def gcs_move_to_bucket(file_id, dest_project_id, skip_gcs=False):
"""Moves a file from its own bucket to the new project_id bucket."""
files_coll = current_app.db()['files']
f = files_coll.find_one(file_id)
if f is None:
raise ValueError('File with _id: {} not found'.format(file_id))
# Check that new backend differs from current one
if f['backend'] != 'gcs':
raise ValueError('Only Google Cloud Storage is supported for now.')
# Move file and variations to the new bucket.
if skip_gcs:
log.warning(
'NOT ACTUALLY MOVING file %s on GCS, just updating MongoDB',
file_id)
else:
from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket
src_project = f['project']
GoogleCloudStorageBucket.copy_to_bucket(f['file_path'], src_project,
dest_project_id)
for var in f.get('variations', []):
GoogleCloudStorageBucket.copy_to_bucket(var['file_path'],
src_project,
dest_project_id)
# Update the file document after moving was successful.
log.info('Switching file %s to project %s', file_id, dest_project_id)
update_result = files_coll.update_one(
{'_id': file_id}, {'$set': {
'project': dest_project_id
}})
if update_result.matched_count != 1:
raise RuntimeError(
'Unable to update file %s in MongoDB: matched_count=%i; modified_count=%i'
% (file_id, update_result.matched_count,
update_result.modified_count))
log.info('Switching file %s: matched_count=%i; modified_count=%i', file_id,
update_result.matched_count, update_result.modified_count)
# Regenerate the links for this file
f['project'] = dest_project_id
generate_all_links(f, now=datetime.datetime.now(tz=bson.tz_util.utc))
def move_to_bucket(file_id: ObjectId, dest_project_id: ObjectId, *, skip_storage=False):
"""Move a file + variations from its own bucket to the new project_id bucket.
:param file_id: ID of the file to move.
:param dest_project_id: Project to move to.
:param skip_storage: If True, the storage bucket will not be touched.
Only use this when you know what you're doing.
"""
files_coll = current_app.db('files')
f = files_coll.find_one(file_id)
if f is None:
raise ValueError(f'File with _id: {file_id} not found')
# Move file and variations to the new bucket.
if skip_storage:
log.warning('NOT ACTUALLY MOVING file %s on storage, just updating MongoDB', file_id)
else:
from pillar.api.file_storage_backends import Bucket
bucket_class = Bucket.for_backend(f['backend'])
src_bucket = bucket_class(str(f['project']))
dst_bucket = bucket_class(str(dest_project_id))
src_blob = src_bucket.get_blob(f['file_path'])
src_bucket.copy_blob(src_blob, dst_bucket)
for var in f.get('variations', []):
src_blob = src_bucket.get_blob(var['file_path'])
src_bucket.copy_blob(src_blob, dst_bucket)
# Update the file document after moving was successful.
# No need to update _etag or _updated, since that'll be done when
# the links are regenerated at the end of this function.
log.info('Switching file %s to project %s', file_id, dest_project_id)
update_result = files_coll.update_one({'_id': file_id},
{'$set': {'project': dest_project_id}})
if update_result.matched_count != 1:
raise RuntimeError(
'Unable to update file %s in MongoDB: matched_count=%i; modified_count=%i' % (
file_id, update_result.matched_count, update_result.modified_count))
log.info('Switching file %s: matched_count=%i; modified_count=%i',
file_id, update_result.matched_count, update_result.modified_count)
# Regenerate the links for this file
f['project'] = dest_project_id
generate_all_links(f, now=utils.utcnow())
def create_groups():
"""Creates the admin/demo/subscriber groups."""
import pprint
group_ids = {}
groups_coll = current_app.db('groups')
for group_name in ['admin', 'demo', 'subscriber']:
if groups_coll.find({'name': group_name}).count():
log.info('Group %s already exists, skipping', group_name)
continue
result = groups_coll.insert_one({'name': group_name})
group_ids[group_name] = result.inserted_id
service.fetch_role_to_group_id_map()
log.info('Created groups:\n%s', pprint.pformat(group_ids))
def update_links(user_id: bson.ObjectId, links: list):
"""Update the links extension property."""
from pymongo.results import UpdateResult
assert isinstance(user_id, bson.ObjectId)
users_coll = current_app.db('users')
result: UpdateResult = users_coll.update_one(
{'_id': user_id},
{'$set': {
'extension_props_public.dillo.links': links
}},
)
if result.matched_count == 0:
raise ValueError(f'User {user_id} not found.')
def get_admin_group_id(project_id: ObjectId) -> ObjectId:
assert isinstance(project_id, ObjectId)
project = current_app.db('projects').find_one({'_id': project_id},
{'permissions': 1})
if not project:
raise ValueError(f'Project {project_id} does not exist.')
# TODO: search through all groups to find the one with the project ID as its name,
# or identify "the admin group" in a different way (for example the group with DELETE rights).
try:
admin_group_id = ObjectId(project['permissions']['groups'][0]['group'])
except KeyError:
raise ValueError(
f'Project {project_id} does not seem to have an admin group')
return admin_group_id
def _manager_project(manager_id, project_url, action):
from pillar.api.utils import str2id
from flamenco import current_flamenco
authentication.force_cli_user()
manager_id = str2id(manager_id)
# Find project
projs_coll = current_app.db()['projects']
proj = projs_coll.find_one({'url': project_url}, projection={'_id': 1})
if not proj:
log.error('Unable to find project url=%s', project_url)
return 1
project_id = proj['_id']
ok = current_flamenco.manager_manager.api_assign_to_project(manager_id, project_id, action)
if not ok:
log.error('Unable to assign manager %s to project %s', manager_id, project_id)
return 1
