def _insert_rna_overrides_task(self, job: dict,
parent_task_selector: dict) -> bson.ObjectId:
# Find the task that is supposed to be the parent of the new task.
tasks_coll = current_flamenco.db('tasks')
if parent_task_selector:
parent_task = tasks_coll.find_one({'job': job['_id'], **parent_task_selector},
projection={'_id': True})
if not parent_task:
raise ValueError('unable to find move-out-of-way task, cannot update this job')
parents_kwargs = {'parents': [parent_task['_id']]}
else:
parents_kwargs = {}
# Construct the new task.
cmd = rna_overrides_command(job)
task_id = self._create_task(job, [cmd], RNA_OVERRIDES_TASK_NAME,
'file-management', priority=80,
status='queued', **parents_kwargs)
self._log.info('Inserted RNA Overrides task %s into job %s', task_id, job['_id'])
# Update existing render tasks to have the new task as parent.
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_many({
'job': job['_id'],
'task_type': 'blender-render',
**parents_kwargs,
}, {'$set': {
'_etag': new_etag,
'_updated': now,
'parents': [task_id],
}})
self._log.debug('Updated %d task parent pointers to %s', result.modified_count, task_id)
return task_id
def merge_project(pid_from: ObjectId, pid_to: ObjectId):
"""Move nodes and files from one project to another.
Note that this may invalidate the nodes, as their node type definition
may differ between projects.
"""
log.info('Moving project contents from %s to %s', pid_from, pid_to)
assert isinstance(pid_from, ObjectId)
assert isinstance(pid_to, ObjectId)
files_coll = current_app.db('files')
nodes_coll = current_app.db('nodes')
# Move the files first. Since this requires API calls to an external
# service, this is more likely to go wrong than moving the nodes.
to_move = files_coll.find({'project': pid_from}, projection={'_id': 1})
log.info('Moving %d files to project %s', to_move.count(), pid_to)
for file_doc in to_move:
fid = file_doc['_id']
log.debug('moving file %s to project %s', fid, pid_to)
move_to_bucket(fid, pid_to)
# Mass-move the nodes.
etag = random_etag()
result = nodes_coll.update_many(
{'project': pid_from},
{'$set': {'project': pid_to,
'_etag': etag,
'_updated': utcnow(),
}}
)
log.info('Moved %d nodes to project %s', result.modified_count, pid_to)
def generate_and_store_token(user_id, days=15, prefix=b'') -> dict:
"""Generates token based on random bits.
NOTE: the returned document includes the plain-text token.
DO NOT STORE OR LOG THIS unless there is a good reason to.
:param user_id: ObjectId of the owning user.
:param days: token will expire in this many days.
:param prefix: the token will be prefixed by these bytes, for easy identification.
:return: the token document with the token in plain text as well as hashed.
"""
if not isinstance(prefix, bytes):
raise TypeError('prefix must be bytes, not %s' % type(prefix))
import secrets
random_bits = secrets.token_bytes(32)
# Use 'xy' as altargs to prevent + and / characters from appearing.
# We never have to b64decode the string anyway.
token = prefix + base64.b64encode(random_bits, altchars=b'xy').strip(b'=')
token_expiry = utcnow() + datetime.timedelta(days=days)
return store_token(user_id, token.decode('ascii'), token_expiry)
def remove_waiting_for_files():
"""Deletes jobs that are stuck in 'waiting-for-files' status.
These jobs are waiting for an external PATCH call to initiate job
compilation, queueing, and execution. If this PATCH call doesn't
come, the job is stuck in this status. After a certain time of
waiting, this function will automatically delete those jobs.
Be sure to add a schedule to the Celery Beat like this:
'remove_waiting_for_files': {
'task': 'flamenco.celery.job_cleanup.remove_waiting_for_files',
'schedule': 3600, # every N seconds
}
"""
age = current_app.config['FLAMENCO_WAITING_FOR_FILES_MAX_AGE'] # type: datetime.timedelta
assert isinstance(age, datetime.timedelta), \
f'FLAMENCO_WAITING_FOR_FILES_MAX_AGE should be a timedelta, not {age!r}'
threshold = utcnow() - age
log.info('Deleting jobs stuck in "waiting-for-files" status that have not been '
'updated since %s', threshold)
jobs_coll = current_flamenco.db('jobs')
result = jobs_coll.delete_many({
'status': 'waiting-for-files',
'_updated': {'$lt': threshold},
})
# No need to delete the tasks, because those jobs don't have any.
log.info('Deleted %d jobs stuck in "waiting-for-files" status', result.deleted_count)
def update_rna_overrides_task(self, job: dict):
"""Update or create an RNA Overrides task of an existing job."""
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one(
{
'job': job['_id'],
'name': RNA_OVERRIDES_TASK_NAME
},
projection={'_id': True})
if not task:
self.insert_rna_overrides_task(job)
return
cmd = rna_overrides_command(job)
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_one(
task, {
'$set': {
'_etag': new_etag,
'_updated': now,
'status': 'queued',
'commands': [cmd.to_dict()],
}
})
self._log.info('Modified %d RNA override task (%s) of job %s',
result.modified_count, task['_id'], job['_id'])
def find_user_to_sync(user_id: bson.ObjectId) -> typing.Optional[SyncUser]:
"""Return user information for syncing badges for a specific user.
Returns None if the user cannot be synced (no 'badge' scope on a token,
or no Blender ID user_id known).
"""
my_log = log.getChild('refresh_single_user')
now = utcnow()
tokens_coll = current_app.db('tokens')
users_coll = current_app.db('users')
token_info = tokens_coll.find_one({
'user': user_id,
'token': {'$exists': True},
'oauth_scopes': 'badge',
'expire_time': {'$gt': now},
})
if not token_info:
my_log.debug('No token with scope "badge" for user %s', user_id)
return None
user_info = users_coll.find_one({'_id': user_id})
# TODO(Sybren): do this filtering in the MongoDB query:
bid_user_ids = [auth_info.get('user_id')
for auth_info in user_info.get('auth', [])
if auth_info.get('provider', '') == 'blender-id' and auth_info.get('user_id')]
if not bid_user_ids:
my_log.debug('No Blender ID user_id for user %s', user_id)
return None
bid_user_id = bid_user_ids[0]
return SyncUser(user_id=user_id, token=token_info['token'], bid_user_id=bid_user_id)
def test_remove_badges(self):
from pillar import badge_sync
from pillar.api.utils import utcnow
# Make sure the user has a badge before getting the 204 No Content from Blender ID.
self._update_badge_expiry(-10, 10)
httpmock.add('GET',
'http://id.local:8001/api/badges/1947/html/s',
body='',
status=204)
with self.app.app_context():
badge_sync.refresh_all_badges(
only_user_id=self.uid1,
timelimit=datetime.timedelta(seconds=4))
expected_expire = utcnow(
) + self.app.config['BLENDER_ID_BADGE_EXPIRY']
# Get directly from MongoDB because JSON doesn't support datetimes.
users_coll = self.app.db('users')
db_user1 = users_coll.find_one(self.uid1)
self.assertEqual('', db_user1['badges']['html'])
margin = datetime.timedelta(minutes=1)
self.assertLess(expected_expire - margin,
db_user1['badges']['expires'])
self.assertGreater(expected_expire + margin,
db_user1['badges']['expires'])
def mark_node_updated(node_id):
"""Uses pymongo to set the node's _updated to "now"."""
now = utcnow()
nodes_coll = current_app.data.driver.db['nodes']
return nodes_coll.update_one({'_id': node_id}, {'$set': {'_updated': now}})
def _update_badge_expiry(self, delta_minutes1, delta_minutes2):
"""Make badges of userN expire in delta_minutesN minutes."""
from pillar.api.utils import utcnow, remove_private_keys
now = utcnow()
# Do the update via Eve so that that flow is covered too.
with self.app.app_context():
users_coll = self.app.db('users')
db_user1 = users_coll.find_one(self.uid1)
db_user1['badges'] = {
'html': 'badge for user 1',
'expires': now + datetime.timedelta(minutes=delta_minutes1)
}
r, _, _, status = self.app.put_internal(
'users', remove_private_keys(db_user1), _id=self.uid1)
self.assertEqual(200, status, r)
with self.app.app_context():
db_user2 = users_coll.find_one(self.uid2)
db_user2['badges'] = {
'html': 'badge for user 2',
'expires': now + datetime.timedelta(minutes=delta_minutes2)
}
r, _, _, status = self.app.put_internal(
'users', remove_private_keys(db_user2), _id=self.uid2)
self.assertEqual(200, status, r)
def api_set_job_priority(self, job_id: ObjectId, new_priority: int):
"""API-level call to updates the job priority."""
assert isinstance(new_priority, int)
self._log.debug('Setting job %s priority to %r', job_id, new_priority)
jobs_coll = current_flamenco.db('jobs')
curr_job = jobs_coll.find_one({'_id': job_id}, projection={'priority': 1})
old_priority = curr_job['priority']
if old_priority == new_priority:
self._log.debug('Job %s is already at priority %r', job_id, old_priority)
return
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
result = jobs_coll.update_one({'_id': job_id},
{'$set': {'priority': new_priority,
'_updated': now,
'_etag': new_etag,
}})
if result.matched_count != 1:
self._log.warning('Matched %d jobs while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
tasks_coll = current_flamenco.db('tasks')
result = tasks_coll.update_many({'job': job_id},
{'$set': {'job_priority': new_priority,
'_updated': now,
'_etag': new_etag,
}})
self._log.debug('Matched %d tasks while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
def api_update_rna_overrides(self, job_id: ObjectId, rna_overrides: typing.List[str]):
"""API-level call to create or update an RNA override task of a Blender Render job."""
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
# Check that the job exists and is a Blender-related job.
job = jobs_coll.find_one({'_id': job_id})
if not job:
self._log.warning('Unable to update RNA overrides of non-existing job %s', job_id)
return None
compiler = job_compilers.construct_job_compiler(job)
if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler):
self._log.warning('Job compiler %r is not an AbstractBlenderJobCompiler, unable '
'to update RNA overrides for job %s of type %r',
type(compiler), job_id, job['job_type'])
return None
# Update the job itself before updating its tasks. Ideally this would happen in the
# same transaction.
# TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+.
job['settings']['rna_overrides'] = rna_overrides
result = jobs_coll.update_one({'_id': job_id}, {'$set': {
'settings.rna_overrides': rna_overrides,
'_updated': now,
'_etag': new_etag,
}})
if result.matched_count != 1:
self._log.warning('Matched %d jobs while setting job %s RNA overrides',
result.matched_count, job_id)
compiler.update_rna_overrides_task(job)
def _task_log_request(self, manager_id: bson.ObjectId, operation: dict):
managers_coll = current_flamenco.db('managers')
managers_coll.update_one({'_id': manager_id}, {
**operation,
'$set': {
'_updated': utcnow(),
'_etag': random_etag(),
},
})
def refresh_all_badges(only_user_id: typing.Optional[bson.ObjectId] = None, *,
dry_run=False,
timelimit: datetime.timedelta):
"""Re-fetch all badges for all users, except when already refreshed recently.
:param only_user_id: Only refresh this user. This is expected to be used
sparingly during manual maintenance / debugging sessions only. It does
fetch all users to refresh, and in Python code skips all except the
given one.
:param dry_run: if True the changes are described in the log, but not performed.
:param timelimit: Refreshing will stop after this time. This allows for cron(-like)
jobs to run without overlapping, even when the number fo badges to refresh
becomes larger than possible within the period of the cron job.
"""
my_log = log.getChild('refresh_all_badges')
# Test the config before we start looping over the world.
badge_expiry = badge_expiry_config()
if not badge_expiry or not isinstance(badge_expiry, datetime.timedelta):
raise ValueError('BLENDER_ID_BADGE_EXPIRY not configured properly, should be a timedelta')
session = _get_requests_session()
deadline = utcnow() + timelimit
num_updates = 0
for user_info in find_users_to_sync():
if utcnow() > deadline:
my_log.info('Stopping badge refresh because the timelimit %s (H:MM:SS) was hit.',
timelimit)
break
if only_user_id and user_info.user_id != only_user_id:
my_log.debug('Skipping user %s', user_info.user_id)
continue
try:
badge_html = fetch_badge_html(session, user_info, 's')
except StopRefreshing:
my_log.error('Blender ID has internal problems, stopping badge refreshing at user %s',
user_info)
break
num_updates += 1
update_badges(user_info, badge_html, badge_expiry, dry_run=dry_run)
my_log.info('Updated badges of %d users%s', num_updates, ' (dry-run)' if dry_run else '')
def create_valid_auth_token(self, user_id, token='token'):
from pillar.api.utils import utcnow
future = utcnow() + datetime.timedelta(days=1)
with self.app.test_request_context():
from pillar.api.utils import authentication as auth
token_data = auth.store_token(user_id, token, future, None)
return token_data
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.admin_uid = self.create_user(24 * 'a', roles={'admin'})
self.uid = self.create_user(24 * 'b', roles={'subscriber'})
from pillar.api.utils import utcnow
self.fake_now = utcnow()
self.fake_now_str = self.fake_now.strftime(RFC1123_DATE_FORMAT)
def _delete_expired_tokens():
"""Deletes tokens that have expired.
For debugging, we keep expired tokens around for a few days, so that we
can determine that a token was expired rather than not created in the
first place. It also grants some leeway in clock synchronisation.
"""
token_coll = current_app.data.driver.db['tokens']
expiry_date = utcnow() - datetime.timedelta(days=7)
result = token_coll.delete_many({'expire_time': {"$lt": expiry_date}})
def change_file_storage_backend(file_id, dest_backend):
"""Given a file document, move it to the specified backend (if not already
there) and update the document to reflect that.
Files on the original backend are not deleted automatically.
"""
dest_backend = str(dest_backend)
file_id = ObjectId(file_id)
# Fetch file document
files_collection = current_app.data.driver.db['files']
f = files_collection.find_one(file_id)
if f is None:
raise ValueError('File with _id: {} not found'.format(file_id))
# Check that new backend differs from current one
if dest_backend == f['backend']:
raise PrerequisiteNotMetError('Destination backend ({}) matches the current backend, we '
'are not moving the file'.format(dest_backend))
# TODO Check that new backend is allowed (make conf var)
# Check that the file has a project; without project, we don't know
# which bucket to store the file into.
try:
project_id = f['project']
except KeyError:
raise PrerequisiteNotMetError('File document does not have a project')
# Ensure that all links are up to date before we even attempt a download.
ensure_valid_link(f)
# Upload file and variations to the new backend
variations = f.get('variations', ())
try:
copy_file_to_backend(file_id, project_id, f, f['backend'], dest_backend)
except requests.exceptions.HTTPError as ex:
# allow the main file to be removed from storage.
if ex.response.status_code not in {404, 410}:
raise
if not variations:
raise PrerequisiteNotMetError('Main file ({link}) does not exist on server, '
'and no variations exist either'.format(**f))
log.warning('Main file %s does not exist; skipping main and visiting variations', f['link'])
for var in variations:
copy_file_to_backend(file_id, project_id, var, f['backend'], dest_backend)
# Generate new links for the file & all variations. This also saves
# the new backend we set here.
f['backend'] = dest_backend
generate_all_links(f, utils.utcnow())
def oauth_callback(provider):
import datetime
from pillar.api.utils.authentication import store_token
from pillar.api.utils import utcnow
next_after_login = session.pop('next_after_login',
None) or url_for('main.homepage')
if current_user.is_authenticated:
log.debug('Redirecting user to %s', next_after_login)
return redirect(next_after_login)
oauth = OAuthSignIn.get_provider(provider)
try:
oauth_user = oauth.callback()
except OAuthCodeNotProvided as e:
log.error(e)
raise wz_exceptions.Forbidden()
if oauth_user.id is None:
log.debug('Authentication failed for user with {}'.format(provider))
return redirect(next_after_login)
# Find or create user
user_info = {
'id': oauth_user.id,
'email': oauth_user.email,
'full_name': ''
}
db_user = find_user_in_db(user_info, provider=provider)
db_id, status = upsert_user(db_user)
# TODO(Sybren): If the user doesn't have any badges, but the access token
# does have 'badge' scope, we should fetch the badges in the background.
if oauth_user.access_token:
# TODO(Sybren): make nr of days configurable, or get from OAuthSignIn subclass.
token_expiry = utcnow() + datetime.timedelta(days=15)
token = store_token(db_id,
oauth_user.access_token,
token_expiry,
oauth_scopes=oauth_user.scopes)
else:
token = generate_and_store_token(db_id)
# Login user
pillar.auth.login_user(token['token'], load_from_db=True)
if provider == 'blender-id' and current_user.is_authenticated:
# Check with Blender ID to update certain user roles.
update_subscription()
log.debug('Redirecting user to %s', next_after_login)
return redirect(next_after_login)
def generate_token(manager_id: str):
manager_oid = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_oid)
# There are three ways in which a user can get here. One is authenticated via
# Authorization header (either Bearer token or Basic token:subtoken), and the
# other is via an already-existing browser session.
# In the latter case it's a redirect from a Flamenco Manager and we need to
# check the timeout and HMAC.
if not request.headers.get('Authorization'):
hasher = current_flamenco.manager_manager.hasher(manager_oid)
if hasher is None:
raise wz_exceptions.InternalServerError(
'Flamenco Manager not linked to this server')
expires = request.args.get('expires', '')
string_to_hash = f'{expires}-{manager_id}'
hasher.update(string_to_hash.encode('utf8'))
actual_hmac = hasher.hexdigest()
query_hmac = request.args.get('hmac', '')
if not hmac.compare_digest(query_hmac, actual_hmac):
raise wz_exceptions.Unauthorized('Bad HMAC')
# Only parse the timestamp after we learned we can trust it.
expire_timestamp = dateutil.parser.parse(expires)
validity_seconds_left = (expire_timestamp - utcnow()).total_seconds()
if validity_seconds_left < 0:
raise wz_exceptions.Unauthorized('Link expired')
if validity_seconds_left > 900:
# Flamenco Manager generates links that are valid for less than a minute, so
# if it's more than 15 minutes in the future, it's bad.
raise wz_exceptions.Unauthorized('Link too far in the future')
user = authentication.current_user()
if not current_flamenco.manager_manager.user_may_use(mngr_doc=manager):
log.warning(
'Account %s called %s for manager %s without access to that manager',
user.user_id, request.url, manager_oid)
raise wz_exceptions.Unauthorized()
jwt = current_flamenco.jwt
if not jwt.usable:
raise wz_exceptions.NotImplemented(
'JWT keystore is not usable at the moment')
log.info('Generating JWT key for user_id=%s manager_id=%s remote_addr=%s',
user.user_id, manager_id, request.remote_addr)
key_for_manager = jwt.generate_key_for_manager(manager_oid, user.user_id)
return Response(key_for_manager, content_type='text/plain')
def find_users_to_sync() -> typing.Iterable[SyncUser]:
"""Return user information of syncable users with badges."""
now = utcnow()
tokens_coll = current_app.db('tokens')
cursor = tokens_coll.aggregate([
# Find all users who have a 'badge' scope in their OAuth token.
{'$match': {
'token': {'$exists': True},
'oauth_scopes': 'badge',
'expire_time': {'$gt': now},
# TODO(Sybren): save real token expiry time but keep checking tokens hourly when they are used!
}},
{'$lookup': {
'from': 'users',
'localField': 'user',
'foreignField': '_id',
'as': 'user'
}},
# Prevent 'user' from being an array.
{'$unwind': {'path': '$user'}},
# Get the Blender ID user ID only.
{'$unwind': {'path': '$user.auth'}},
{'$match': {'user.auth.provider': 'blender-id'}},
# Only select those users whose badge doesn't exist or has expired.
{'$match': {
'user.badges.expires': {'$not': {'$gt': now}}
}},
# Make sure that the badges that expire last are also refreshed last.
{'$sort': {'user.badges.expires': 1}},
# Reduce the document to the info we're after.
{'$project': {
'token': True,
'user._id': True,
'user.auth.user_id': True,
}},
])
log.debug('Aggregating tokens and users')
for user_info in cursor:
log.debug('User %s has badges %s',
user_info['user']['_id'], user_info['user'].get('badges'))
yield SyncUser(
user_id=user_info['user']['_id'],
token=user_info['token'],
bid_user_id=user_info['user']['auth']['user_id'])
def generate_token(manager_id: str):
manager_oid = str2id(manager_id)
manager = mongo.find_one_or_404('flamenco_managers', manager_oid)
# There are three ways in which a user can get here. One is authenticated via
# Authorization header (either Bearer token or Basic token:subtoken), and the
# other is via an already-existing browser session.
# In the latter case it's a redirect from a Flamenco Manager and we need to
# check the timeout and HMAC.
if not request.headers.get('Authorization'):
hasher = current_flamenco.manager_manager.hasher(manager_oid)
if hasher is None:
raise wz_exceptions.InternalServerError('Flamenco Manager not linked to this server')
expires = request.args.get('expires', '')
string_to_hash = f'{expires}-{manager_id}'
hasher.update(string_to_hash.encode('utf8'))
actual_hmac = hasher.hexdigest()
query_hmac = request.args.get('hmac', '')
if not hmac.compare_digest(query_hmac, actual_hmac):
raise wz_exceptions.Unauthorized('Bad HMAC')
# Only parse the timestamp after we learned we can trust it.
expire_timestamp = dateutil.parser.parse(expires)
validity_seconds_left = (expire_timestamp - utcnow()).total_seconds()
if validity_seconds_left < 0:
raise wz_exceptions.Unauthorized('Link expired')
if validity_seconds_left > 900:
# Flamenco Manager generates links that are valid for less than a minute, so
# if it's more than 15 minutes in the future, it's bad.
raise wz_exceptions.Unauthorized('Link too far in the future')
user = authentication.current_user()
if not current_flamenco.manager_manager.user_may_use(mngr_doc=manager):
log.warning(
'Account %s called %s for manager %s without access to that manager',
user.user_id, request.url, manager_oid)
raise wz_exceptions.Unauthorized()
jwt = current_flamenco.jwt
if not jwt.usable:
raise wz_exceptions.NotImplemented('JWT keystore is not usable at the moment')
log.info('Generating JWT key for user_id=%s manager_id=%s remote_addr=%s',
user.user_id, manager_id, request.remote_addr)
key_for_manager = jwt.generate_key_for_manager(manager_oid, user.user_id)
return Response(key_for_manager, content_type='text/plain')
def _insert_rna_overrides_task(
self, job: dict, parent_task_selector: dict) -> bson.ObjectId:
# Find the task that is supposed to be the parent of the new task.
tasks_coll = current_flamenco.db('tasks')
if parent_task_selector:
parent_task = tasks_coll.find_one(
{
'job': job['_id'],
**parent_task_selector
},
projection={'_id': True})
if not parent_task:
raise ValueError(
'unable to find move-out-of-way task, cannot update this job'
)
parents_kwargs = {'parents': [parent_task['_id']]}
else:
parents_kwargs = {}
# Construct the new task.
cmd = rna_overrides_command(job)
task_id = self._create_task(job, [cmd],
RNA_OVERRIDES_TASK_NAME,
'file-management',
priority=80,
status='queued',
**parents_kwargs)
self._log.info('Inserted RNA Overrides task %s into job %s', task_id,
job['_id'])
# Update existing render tasks to have the new task as parent.
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_many(
{
'job': job['_id'],
'task_type': 'blender-render',
**parents_kwargs,
}, {
'$set': {
'_etag': new_etag,
'_updated': now,
'parents': [task_id],
}
})
self._log.debug('Updated %d task parent pointers to %s',
result.modified_count, task_id)
return task_id
def sync(email: str = '', sync_all: bool = False, go: bool = False):
if bool(email) == bool(sync_all):
raise ValueError('Use either --user or --all.')
if email:
users_coll = current_app.db('users')
db_user = users_coll.find_one({'email': email},
projection={'_id': True})
if not db_user:
raise ValueError(f'No user with email {email!r} found')
specific_user = db_user['_id']
else:
specific_user = None
if not go:
log.info('Performing dry-run, not going to change the user database.')
start_time = utcnow()
badge_sync.refresh_all_badges(specific_user,
dry_run=not go,
timelimit=datetime.timedelta(hours=1))
end_time = utcnow()
log.info('%s took %s (H:MM:SS)',
'Updating user badges' if go else 'Dry-run',
end_time - start_time)
def check_download_property_update(item, original):
"""Update node if properties.download was updated.
In particular, reset the downloads_latest to 0 and update hotness.
The hotness is updated based on _updated and not _created.
"""
if 'download' not in item['properties'] or 'download' not in original[
'properties']:
return
if item['properties']['download'] != original['properties'].get(
'download'):
item['properties']['downloads_latest'] = 0
# Update the upload timestamp
item['properties']['download_updated'] = utcnow()
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{
'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
def move_to_bucket(file_id: ObjectId, dest_project_id: ObjectId, *, skip_storage=False):
"""Move a file + variations from its own bucket to the new project_id bucket.
:param file_id: ID of the file to move.
:param dest_project_id: Project to move to.
:param skip_storage: If True, the storage bucket will not be touched.
Only use this when you know what you're doing.
"""
files_coll = current_app.db('files')
f = files_coll.find_one(file_id)
if f is None:
raise ValueError(f'File with _id: {file_id} not found')
# Move file and variations to the new bucket.
if skip_storage:
log.warning('NOT ACTUALLY MOVING file %s on storage, just updating MongoDB', file_id)
else:
from pillar.api.file_storage_backends import Bucket
bucket_class = Bucket.for_backend(f['backend'])
src_bucket = bucket_class(str(f['project']))
dst_bucket = bucket_class(str(dest_project_id))
src_blob = src_bucket.get_blob(f['file_path'])
src_bucket.copy_blob(src_blob, dst_bucket)
for var in f.get('variations', []):
src_blob = src_bucket.get_blob(var['file_path'])
src_bucket.copy_blob(src_blob, dst_bucket)
# Update the file document after moving was successful.
# No need to update _etag or _updated, since that'll be done when
# the links are regenerated at the end of this function.
log.info('Switching file %s to project %s', file_id, dest_project_id)
update_result = files_coll.update_one({'_id': file_id},
{'$set': {'project': dest_project_id}})
if update_result.matched_count != 1:
raise RuntimeError(
'Unable to update file %s in MongoDB: matched_count=%i; modified_count=%i' % (
file_id, update_result.matched_count, update_result.modified_count))
log.info('Switching file %s: matched_count=%i; modified_count=%i',
file_id, update_result.matched_count, update_result.modified_count)
# Regenerate the links for this file
f['project'] = dest_project_id
generate_all_links(f, now=utils.utcnow())
def _compute_token_expiry(token_expires_string):
"""Computes token expiry based on current time and BlenderID expiry.
Expires our side of the token when either the BlenderID token expires,
or in one hour. The latter case is to ensure we periodically verify
the token.
"""
# requirement is called python-dateutil, so PyCharm doesn't find it.
# noinspection PyPackageRequirements
from dateutil import parser
blid_expiry = parser.parse(token_expires_string)
blid_expiry = blid_expiry.astimezone(tz_util.utc)
our_expiry = utcnow() + datetime.timedelta(hours=1)
return min(blid_expiry, our_expiry)
def resume_job_archiving():
"""Resumes archiving of jobs that are stuck in status "archiving".
Finds all jobs in status "archiving" that is older than one day and
calls archive_job with each job.
"""
age = current_app.config['FLAMENCO_RESUME_ARCHIVING_AGE']
jobs_coll = current_flamenco.db('jobs')
archiving = jobs_coll.find({
'status': 'archiving',
'_updated': {'$lte': utcnow() - age},
})
log.info('Resume archiving %d jobs', archiving.count())
for job in archiving:
log.debug('Resume archiving job %s', job['_id'])
archive_job.delay(str(job['_id']))
def expire_all_project_links(project_uuid):
"""Expires all file links for a certain project without refreshing.
This is just for testing.
"""
import datetime
from pillar.api.utils import utcnow
files_collection = current_app.data.driver.db['files']
expires = utcnow() - datetime.timedelta(days=1)
result = files_collection.update_many(
{'project': ObjectId(project_uuid)},
{'$set': {'link_expires': expires}}
)
print('Expired %i links' % result.matched_count)
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.private_pid, _ = self.ensure_project_exists(project_overrides={
'_id': '5672beecc0261b2005ed1a34',
'is_private': True,
})
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{
'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
self.asset_node_id = self.create_node({
'name':
'Just a node name',
'project':
self.pid,
'description':
'',
'node_type':
'asset',
'user':
self.uid,
'_created':
self.fake_now - timedelta(weeks=52),
'properties':
base_props,
})
def resume_job_archiving():
"""Resumes archiving of jobs that are stuck in status "archiving".
Finds all jobs in status "archiving" that is older than one day and
calls archive_job with each job.
"""
age = current_app.config['FLAMENCO_RESUME_ARCHIVING_AGE']
jobs_coll = current_flamenco.db('jobs')
archiving = jobs_coll.find({
'status': 'archiving',
'_updated': {
'$lte': utcnow() - age
},
})
log.info('Resume archiving %d jobs', archiving.count())
for job in archiving:
log.debug('Resume archiving job %s', job['_id'])
archive_job.delay(str(job['_id']))
def index():
api = system_util.pillar_api()
# Get all projects, except the home project.
projects_user = Project.all({
'where': {'user': current_user.objectid,
'category': {'$ne': 'home'}},
'sort': '-_created'
}, api=api)
show_deleted_projects = request.args.get('deleted') is not None
if show_deleted_projects:
timeframe = utcnow() - datetime.timedelta(days=31)
projects_deleted = Project.all({
'where': {'user': current_user.objectid,
'category': {'$ne': 'home'},
'_deleted': True,
'_updated': {'$gt': timeframe}},
'sort': '-_created'
}, api=api)
else:
projects_deleted = {'_items': []}
projects_shared = Project.all({
'where': {'user': {'$ne': current_user.objectid},
'permissions.groups.group': {'$in': current_user.groups},
'is_private': True},
'sort': '-_created',
'embedded': {'user': 1},
}, api=api)
# Attach project images
for project_list in (projects_user, projects_deleted, projects_shared):
utils.mass_attach_project_pictures(project_list['_items'], api=api, header=False)
return render_template(
'projects/index_dashboard.html',
gravatar=utils.gravatar(current_user.email, size=128),
projects_user=projects_user['_items'],
projects_deleted=projects_deleted['_items'],
projects_shared=projects_shared['_items'],
show_deleted_projects=show_deleted_projects,
api=api)
def test_rewatch_after_done(self):
from pillar.api.utils import utcnow
self.create_video_and_set_progress(630, 100)
# Re-watching should keep the 'done' key.
another_fake_now = utcnow()
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = another_fake_now
self.set_progress(444, 70)
progress = self.get_progress()
self.assertEqual(
{
'progress_in_sec': 444,
'progress_in_percent': 70,
'done': True,
'last_watched': another_fake_now.strftime(RFC1123_DATE_FORMAT)
}, progress)
def update_badges(user_info: SyncUser, badge_html: str, badge_expiry: datetime.timedelta,
*, dry_run: bool):
my_log = log.getChild('update_badges')
users_coll = current_app.db('users')
update = {'badges': {
'html': badge_html,
'expires': utcnow() + badge_expiry,
}}
my_log.info('Updating badges HTML for Blender ID %s, user %s',
user_info.bid_user_id, user_info.user_id)
if dry_run:
return
result = users_coll.update_one({'_id': user_info.user_id},
{'$set': update})
if result.matched_count != 1:
my_log.warning('Unable to update badges for user %s', user_info.user_id)
def api_update_rna_overrides(self, job_id: ObjectId,
rna_overrides: typing.List[str]):
"""API-level call to create or update an RNA override task of a Blender Render job."""
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
# Check that the job exists and is a Blender-related job.
job = jobs_coll.find_one({'_id': job_id})
if not job:
self._log.warning(
'Unable to update RNA overrides of non-existing job %s',
job_id)
return None
compiler = job_compilers.construct_job_compiler(job)
if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler):
self._log.warning(
'Job compiler %r is not an AbstractBlenderJobCompiler, unable '
'to update RNA overrides for job %s of type %r',
type(compiler), job_id, job['job_type'])
return None
# Update the job itself before updating its tasks. Ideally this would happen in the
# same transaction.
# TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+.
job['settings']['rna_overrides'] = rna_overrides
result = jobs_coll.update_one({'_id': job_id}, {
'$set': {
'settings.rna_overrides': rna_overrides,
'_updated': now,
'_etag': new_etag,
}
})
if result.matched_count != 1:
self._log.warning(
'Matched %d jobs while setting job %s RNA overrides',
result.matched_count, job_id)
compiler.update_rna_overrides_task(job)
def update_rna_overrides_task(self, job: dict):
"""Update or create an RNA Overrides task of an existing job."""
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one({'job': job['_id'], 'name': RNA_OVERRIDES_TASK_NAME},
projection={'_id': True})
if not task:
self.insert_rna_overrides_task(job)
return
cmd = rna_overrides_command(job)
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_one(task, {'$set': {
'_etag': new_etag,
'_updated': now,
'status': 'queued',
'commands': [cmd.to_dict()],
}})
self._log.info('Modified %d RNA override task (%s) of job %s',
result.modified_count, task['_id'], job['_id'])
def test_resume_archiving(self, mock_archive_job):
from flamenco.celery import job_archival
now = utcnow()
# 1 day old in status archiving. Should be resumed.
self.force_job_status('archiving', self.job1_id)
self.set_job_updated(now - datetime.timedelta(days=1), self.job1_id)
# In archiving status but too new. Should *not* be resumed.
self.force_job_status('archiving', self.job2_id)
self.set_job_updated(now - datetime.timedelta(hours=23), self.job2_id)
# 1 day old but in wrong status. Should *not* be resumed.
self.set_job_updated(now - datetime.timedelta(days=1), self.job3_id)
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = now
job_archival.resume_job_archiving()
mock_archive_job.delay.assert_called_once()
mock_archive_job.delay.assert_called_with(str(self.job1_id))
def handle_task_update_batch(manager_id, task_updates):
"""Performs task updates.
Task status changes are generally always accepted. The only exception is when the
task ID is contained in 'tasks_to_cancel'; in that case only a transition to either
'canceled', 'completed' or 'failed' is accepted.
:returns: tuple (total nr of modified tasks, handled update IDs)
"""
if not task_updates:
return 0, []
import dateutil.parser
from pillar.api.utils import str2id
from flamenco import current_flamenco, eve_settings
log.debug('Received %i task updates from manager %s', len(task_updates), manager_id)
tasks_coll = current_flamenco.db('tasks')
logs_coll = current_flamenco.db('task_logs')
valid_statuses = set(eve_settings.tasks_schema['status']['allowed'])
handled_update_ids = []
total_modif_count = 0
for task_update in task_updates:
# Check that this task actually belongs to this manager, before we accept any updates.
update_id = str2id(task_update['_id'])
task_id = str2id(task_update['task_id'])
task_info = tasks_coll.find_one({'_id': task_id},
projection={'manager': 1, 'status': 1, 'job': 1})
# For now, we just ignore updates to non-existing tasks. Someone might have just deleted
# one, for example. This is not a reason to reject the entire batch.
if task_info is None:
log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring',
manager_id, task_id)
handled_update_ids.append(update_id)
continue
if task_info['manager'] != manager_id:
log.warning('Manager %s sent update for task %s which belongs to other manager %s',
manager_id, task_id, task_info['manager'])
continue
if task_update.get('received_on_manager'):
received_on_manager = dateutil.parser.parse(task_update['received_on_manager'])
else:
# Fake a 'received on manager' field; it really should have been in the JSON payload.
received_on_manager = utcnow()
# Store the log for this task, allowing for duplicate log reports.
#
# NOTE: is deprecated and will be removed in a future version of Flamenco;
# only periodically send the last few lines of logging in 'log_tail' and
# store the entire log on the Manager itself.
task_log = task_update.get('log')
if task_log:
log_doc = {
'_id': update_id,
'task': task_id,
'received_on_manager': received_on_manager,
'log': task_log
}
logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True)
# Modify the task, and append the log to the logs collection.
updates = {
'task_progress_percentage': task_update.get('task_progress_percentage', 0),
'current_command_index': task_update.get('current_command_index', 0),
'command_progress_percentage': task_update.get('command_progress_percentage', 0),
'_updated': received_on_manager,
'_etag': random_etag(),
}
new_status = determine_new_task_status(manager_id, task_id, task_info,
task_update.get('task_status'), valid_statuses)
if new_status:
updates['status'] = new_status
new_activity = task_update.get('activity')
if new_activity:
updates['activity'] = new_activity
worker = task_update.get('worker')
if worker:
updates['worker'] = worker
metrics_timing = task_update.get('metrics', {}).get('timing')
if metrics_timing:
updates['metrics.timing'] = metrics_timing
fbw = task_update.get('failed_by_workers')
if fbw is not None:
updates['failed_by_workers'] = fbw
# Store the last lines of logging on the task itself.
task_log_tail: str = task_update.get('log_tail')
if not task_log_tail and task_log:
task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:])
if task_log_tail:
updates['log'] = task_log_tail
result = tasks_coll.update_one({'_id': task_id}, {'$set': updates})
total_modif_count += result.modified_count
handled_update_ids.append(update_id)
# Update the task's job after updating the task itself.
if new_status:
current_flamenco.job_manager.update_job_after_task_status_change(
task_info['job'], task_id, new_status)
return total_modif_count, handled_update_ids
