def put(self):
assert self.request.dc.id == DefaultDc().id
ser = UpdateSerializer(self.request, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(self.request,
ser.errors,
task_id=self.task_id,
dc_bound=False)
version = ser.object['version']
from core.version import __version__ as mgmt_version
if version == ('v' + mgmt_version):
raise PreconditionRequired('System is already up-to-date')
lock = TaskLock(self._lock_key, desc='System task')
if not lock.acquire(self.task_id, timeout=7200, save_reverse=False):
raise TaskIsAlreadyRunning
try:
return self._update(version,
key=ser.object.get('key'),
cert=ser.object.get('cert'))
finally:
lock.delete(fail_silently=True, delete_reverse=False)
def update(cls, task_id, repo):
lock = TaskLock(cls.LOCK_KEY,
desc='Image repository %s update' % repo.name)
if not lock.acquire(task_id, timeout=60, save_reverse=False):
raise TaskIsAlreadyRunning
try:
return cls._update(task_id, repo)
finally:
lock.delete(fail_silently=True, delete_reverse=False)
def inner(*args, **kwargs):
if bound_task:
params = args[
1:] # The first parameter is a celery task/request object
else:
params = args
task_id = params[0]
task_name = fun.__name__
lock_keys = [task_name]
lock_keys.extend(str(params[i]) for i in key_args)
lock_keys.extend(str(kwargs[i]) for i in key_kwargs)
task_lock = TaskLock(':'.join(lock_keys),
desc='Task %s' % task_name)
def acquire_lock():
return task_lock.acquire(task_id,
timeout=timeout,
save_reverse=False)
if not acquire_lock():
existing_lock = task_lock.get()
if wait_for_release:
logger.warn(
'Task %s(%s, %s) must wait (%s), because another task %s is already running',
task_name, args, kwargs, timeout or 'forever',
existing_lock)
wait = 0
while wait < timeout:
sleep(3)
wait += 3
if acquire_lock():
break
else:
logger.warn(
'Task %s(%s, %s) will not run, because another task %s is still running and we have'
' waited for too long (%s)', task_name, args,
kwargs, existing_lock, wait)
return
else:
logger.warn(
'Task %s(%s, %s) will not run, because another task %s is already running',
task_name, args, kwargs, existing_lock)
return
try:
return fun(*args, **kwargs)
finally:
task_lock.delete(fail_silently=True, delete_reverse=False)
def inner(*args, **kwargs):
if bound:
params = args[1:] # The first parameter is a "self" object
else:
params = args
if base_name:
fun_name = base_name
else:
fun_name = fun.__name__
lock_keys = [fun_name]
lock_keys.extend(str(params[i]) for i in key_args)
lock_keys.extend(str(kwargs[i]) for i in key_kwargs)
task_lock = TaskLock(':'.join(lock_keys),
desc='Function %s' % fun_name)
def acquire_lock():
return task_lock.acquire(time(),
timeout=timeout,
save_reverse=False)
if not acquire_lock():
if wait_for_release:
logger.warn(
'Function %s(%s, %s) must wait (%s), because another function is already running',
fun_name, args, kwargs, timeout or 'forever')
wait = 0
while wait < timeout:
sleep(1)
wait += 1
if acquire_lock():
break
else:
logger.warn(
'Function %s(%s, %s) will not run, because another function is still running and '
'we have waited for too long (%s)', fun_name, args,
kwargs, wait)
return
else:
logger.warn(
'Function %s(%s, %s) will not run, because another function is already running',
fun_name, args, kwargs)
return
try:
return fun(*args, **kwargs)
finally:
task_lock.delete(fail_silently=True, delete_reverse=False)
def __call__(self, *args, **kwargs):
self.logger = get_task_logger('que.mgmt')
task = 'MgmtTask %s%s' % (self.name, args[:2])
tidlock = kwargs.pop('tidlock', None)
check_user_tasks = kwargs.pop('check_user_tasks', False)
kwargs.pop('cache_result', None)
kwargs.pop('cache_timeout', None)
kwargs.pop('nolog', None)
tid = self.request.id
if tidlock:
task_lock = TaskLock(tidlock, desc=task, logger=self.logger)
else:
task_lock = NoLock()
try:
if check_user_tasks: # Wait for task to appear in UserTasks - bug #chili-618
UserTasks.check(
tid, logger=self.logger
) # Will raise an exception in case the task does not show up
task_lock.task_check(
) # Will raise an exception in case the lock does not exist
return super(MgmtTask, self).__call__(tid, *args,
**kwargs) # run()
finally:
task_lock.delete()
def __call__(self, cmd, *args, **kwargs):
self.logger = get_task_logger('que.tasks')
self.all_done = False
task = 'Task %s("%s")' % (self.name, cmd)
lock = kwargs.pop('lock', False)
block = kwargs.pop('block', None)
check_user_tasks = kwargs.pop('check_user_tasks', False)
tid = self.request.id
blocked = False
if lock:
task_lock = TaskLock(lock, desc=task, logger=self.logger)
else:
task_lock = NoLock()
try:
if check_user_tasks: # Wait for task to appear in UserTasks - bug #chili-618
UserTasks.check(
tid, logger=self.logger
) # Will raise an exception in case the task does not show up
task_lock.task_check(
) # Will raise an exception in case the lock does not exist
if block and redis.exists(block):
blocked = True
self.retry(exc=TaskRetry(None)) # Will raise special exception
return super(MetaTask, self).__call__(cmd, *args,
**kwargs) # run()
finally:
if not blocked: # Lock must _not_ be deleted when failing on retry
task_lock.delete()
def put(self):
assert self.request.dc.id == DefaultDc().id
ser = UpdateSerializer(self.request, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(self.request,
ser.errors,
task_id=self.task_id,
dc_bound=False)
node = self.node
version = ser.object['version']
node_version = node.system_version
if not (isinstance(node_version, text_type) and node_version):
raise NodeIsNotOperational(
'Node version information could not be retrieved')
if version == ('v' + node.system_version):
raise PreconditionRequired('Node is already up-to-date')
if node.status != node.OFFLINE:
raise NodeIsNotOperational(
'Unable to perform update on node that is not in OFFLINE state!'
)
lock = TaskLock(self._lock_key, desc='System task')
if not lock.acquire(self.task_id, timeout=7200, save_reverse=False):
raise TaskIsAlreadyRunning
try:
return self._update(version,
key=ser.object.get('key'),
cert=ser.object.get('cert'))
finally:
lock.delete(fail_silently=True, delete_reverse=False)
def log_task_callback(task_id,
task_status=states.REVOKED,
cleanup=True,
detail='revoked',
sender_name='???',
send_forever=True):
"""
Mark task status with task_status and create log callback task. USE with caution!
"""
user_id, owner_id = user_owner_ids_from_task_id(task_id)
# Internal user task
if owner_id == TASK_USER:
logger.debug(
'Task %s[%s] %s in log_task_callback :: Internal task - skipping',
sender_name, task_id, detail)
return None
# If a lock created by this task still exists we need to remove it now - Bug #chili-592
if cleanup:
lock_key = TaskLock.get_lock_key_from_value(task_id)
if lock_key:
TaskLock(lock_key,
desc='LogTask %s' % task_id).delete(check_value=task_id,
fail_silently=True)
# Task info from cache
task_info = UserTasks(owner_id).get(task_id)
if task_info is None:
logger.critical(
'Task %s[%s] %s in log_task_callback :: Task not found in UserTasks',
sender_name, task_id, detail)
return None
# Kwargs for logtask
task_info['task_status'] = task_status
task_info['cleanup'] = cleanup
# Create log task on mgmt
result = {
'detail': detail,
'meta': {
'cb_name': LOGTASK,
'msg': task_info.get('msg', ''),
'apiview': task_info.get('apiview', {})
}
}
task_params = {
'args': (result, task_id),
'kwargs': task_info,
'queue': Q_MGMT,
'expires': None, # This is callback -> never expire
'task_id': task_id_from_task_id(task_id),
}
if send_forever:
t = send_task_forever(task_id, LOGTASK, **task_params)
else:
t = cq.send_task(LOGTASK, **task_params)
if t:
logger.info(
'Task %s[%s] %s in log_task_callback :: Created logtask %s',
sender_name, task_id, detail, t.id)
else:
logger.error(
'Task %s[%s] %s in log_task_callback :: Failed to create logtask',
sender_name, task_id, detail)
return t
def get_lock(cls, tidlock_key, **kwargs):
return TaskLock(cls.TIDLOCK_KEY_TEMPLATE % tidlock_key, **kwargs)
def call(self, request, owner_id, args, kwargs=None, meta=None, tt=TT_MGMT, tg=TG_DC_BOUND,
tidlock=None, tidlock_timeout=None, cache_result=None, cache_timeout=None, expires=EXPIRES,
nolog=False, ping_worker=True, check_user_tasks=True):
"""
Creates task in mgmt queue.
Returns task_id, error_message and cached_result (if any).
"""
if kwargs is None:
kwargs = {}
if meta is None:
meta = {}
tid = task_id_from_request(request, owner_id=owner_id, tt=tt, tg=tg)
task = 'MgmtTask %s[%s]%s' % (self.name, tid, args[:2])
tidlock_acquired = False
if cache_result:
cache_result = self.CACHE_KEY_TEMPLATE % cache_result
result = redis.get(cache_result)
if result is not None:
try:
res = pickle.loads(result)
except pickle.UnpicklingError:
logger.critical('%s could not parse cache_result "%s"', task, cache_result)
else:
return None, None, res
if ping_worker:
if not ping(Q_MGMT, timeout=ping_worker, count=2):
return None, 'Task queue worker (%s) is not responding!' % queue_to_hostnames(Q_MGMT), None
try:
if tidlock:
tidlock = self.TIDLOCK_KEY_TEMPLATE % tidlock
task_lock = TaskLock(tidlock, desc=task)
_tid = task_lock.get()
if _tid:
logger.info('%s found the same pending task %s :)', task, _tid)
return _tid, None, None
if tidlock_timeout is None:
tidlock_timeout = expires
tidlock_acquired = task_lock.acquire(tid, timeout=tidlock_timeout)
if not tidlock_acquired:
return None, 'MgmtTask did not acquire lock', None
kwargs['meta'] = meta
kwargs['tidlock'] = tidlock
kwargs['cache_result'] = cache_result
kwargs['cache_timeout'] = cache_timeout
kwargs['nolog'] = nolog
kwargs['check_user_tasks'] = check_user_tasks
# Run task
t = self.apply_async(args=args, kwargs=kwargs, queue=Q_MGMT, task_id=tid,
expires=expires, add_to_parent=False)
except Exception as e:
logger.exception(e)
logger.error('%s could not be created (%s)', task, e)
if tidlock_acquired: # tidlock_acquired will be True, only if task_lock exists
# noinspection PyUnboundLocalVariable
task_lock.delete(fail_silently=True, premature=True)
return None, e, None
else:
if nolog:
logger.debug('%s created', task)
else:
logger.info('%s created', task)
return t.id, None, None
def execute(request,
owner_id,
cmd,
stdin=None,
meta=None,
callback=None,
lock=None,
lock_timeout=None,
queue=None,
expires=EXPIRES,
tt=TT_EXEC,
tg=TG_DC_BOUND,
nolog=False,
ping_worker=True,
check_user_tasks=True,
block_key=None):
"""
_execute task wrapper. This just looks better and does some locking.
Returns task_id and error_message
"""
task_id = task_id_from_request(request, owner_id=owner_id, tt=tt, tg=tg)
task = 'Task %s[%s]("%s")' % (_execute.name, task_id, cmd)
lock_key = lock
lock_acquired = False
if meta is None:
meta = {}
if ping_worker and queue:
# callback=None means, that an automatic log task callback will run
if callback is not False and queue != Q_MGMT:
queues = [queue, Q_MGMT]
else:
queues = [queue]
for q in queues:
if not ping(q, timeout=ping_worker, count=2):
return None, 'Task queue worker (%s) is not responding!' % queue_to_hostnames(
q)
try:
if lock_key:
if lock_timeout is None:
lock_timeout = expires
lock_key = KEY_PREFIX + lock
task_lock = TaskLock(lock_key, desc=task)
lock_acquired = task_lock.acquire(task_id, timeout=lock_timeout)
if not lock_acquired:
return task_id, 'Task did not acquire lock'
meta['nolog'] = nolog
args = (cmd, stdin)
kwargs = {
'meta': meta,
'callback': callback,
'lock': lock_key,
'block': block_key,
'check_user_tasks': check_user_tasks
}
# Run task
task = _execute.apply_async(args=args,
kwargs=kwargs,
queue=queue,
task_id=task_id,
expires=expires,
add_to_parent=False)
except Exception as e:
logger.exception(e)
logger.error('%s could not be created (%s)', task, e)
if lock_acquired: # lock_acquired will be True, only if task_lock exists
# noinspection PyUnboundLocalVariable
task_lock.delete(fail_silently=True, premature=True)
return None, e
else:
if nolog:
logger.debug('%s created', task)
else:
logger.info('%s created', task)
return task.id, None