def __init__(self, agent_id, ctxt=None,
resultcode=0, result=None, details=None):
self.agent_id = agent_id
self.details = details
self.agent_time = int(timeutils.realnow())
self.expire = ctxt.get('expire', 0)
super(AgentRpcResult, self).__init__(resultcode, result)
def rpcfinishtime(starttime=None):
rpc_conf = rabbit_conf
if not starttime:
starttime = int(timeutils.realnow())
offset_time = rpc_conf.rpc_send_timeout * (rpc_conf.rpc_send_retry + 1)
timeout = offset_time + 5
return starttime + timeout, timeout + 1
def realnowint(after=0):
if after:
def t():
return int(timeutils.realnow()) + after
return t
else:
return int(timeutils.realnow())
def create_asyncrequest(body):
"""async request use this to create a new request
argv in body
request_time: unix time in seconds that client send async request
finishtime: unix time in seconds that work shoud be finished after this time
deadline: unix time in seconds that work will igonre after this time
expire: respone expire time
"""
request_time = int(timeutils.realnow())
expire = body.pop('expire', 0)
if expire < 0:
raise InvalidArgument('Async argv expire less thne 0')
try:
client_request_time = int(body.pop('request_time'))
except KeyError:
raise InvalidArgument('Async request need argument request_time')
except TypeError:
raise InvalidArgument(
'request_time is not int of time or no request_time found')
offset_time = request_time - client_request_time
if abs(offset_time) > 5:
raise InvalidArgument(
'The offset time between send and receive is %d' % offset_time)
finishtime = body.pop('finishtime', None)
if finishtime:
finishtime = int(finishtime) + offset_time
else:
finishtime = rpcfinishtime(request_time)[0]
if finishtime - request_time < 3:
raise InvalidArgument('Job can not be finished in 3 second')
deadline = body.pop('deadline', None)
if deadline:
deadline = int(deadline) + offset_time - 1
else:
# deadline = rpcdeadline(finishtime)
deadline = finishtime + 5
if deadline - finishtime < 3:
raise InvalidArgument(
'Job deadline must at least 3 second after finishtime')
request_id = uuidutils.generate_uuid()
# req.environ[manager_common.ENV_REQUEST_ID] = request_id
new_request = AsyncRequest(request_id=request_id,
request_time=request_time,
finishtime=finishtime,
deadline=deadline,
expire=expire)
return new_request
def add_agent(self, body):
agent = Agent()
try:
agent.host = validators['type:hostname'](body.pop('host'))
agent.agent_type = body.pop('agent_type', None)
if agent.agent_type is None or len(agent.agent_type) > 64:
raise ValueError('Agent type info over size')
if body.get('ports_range', None) is not None:
agent.ports_range = jsonutils.dumps(body.pop('ports_range'))
agent.memory = int(body.pop('memory'))
agent.cpu = int(body.pop('cpu'))
agent.disk = int(body.pop('disk'))
endpoints = validateutils.validate_endpoints(body.pop('endpoints', []))
except KeyError as e:
raise InvalidArgument('Can not find argument: %s' % e.message)
except ValueError as e:
raise InvalidArgument('Argument value type error: %s' % e.message)
agent.create_time = timeutils.realnow()
session = self.session()
with self._lock_all_agents():
host_filter = and_(Agent.host == agent.host, Agent.status > manager_common.DELETED)
if model_count_with_key(session, Agent.host, filter=host_filter) > 0:
raise exceptions.AgentHostExist('Agent with host %s alreday eixst' % agent.host)
agent_id = model_autoincrement_id(session, Agent.agent_id)
with session.begin():
agent.agent_id = agent_id
# if agent.endpoints:
# for endpoint in agent.endpoints:
# endpoint.agent_id = agent_id
session.add(agent)
session.flush()
if endpoints:
for endpoint in endpoints:
session.add(AgentEndpoint(agent_id=agent_id, endpoint=endpoint))
session.flush()
# add new agent_id to cache all agent_id
# if not self.client.sadd(self.ALL_AGENTS_KEY, str(agent.agent_id)):
if not self.client.zadd(self.ALL_AGENTS_KEY, int(time.time()), str(agent.agent_id)):
raise exceptions.CacheStoneError('Cant not add agent_id to redis, key %s' % self.ALL_AGENTS_KEY)
_endpoints = [e.endpoint for e in agent.endpoints]
LOG.info('New anget with endpoints %s' % ','.join(_endpoints))
return agent
def t():
return int(timeutils.realnow()) + after
def _send(self,
target,
ctxt,
message,
wait_for_reply=None,
timeout=None,
notify=False,
retry=None):
context = ctxt
msg = message
# Set namespace for dispatcher
ctxt.update({
'namespace': target.namespace,
'casttime': int(realnow())
})
if wait_for_reply:
# ctxt.update({'reply': True})
msg_id = uuid.uuid4().hex
msg.update({'_msg_id': msg_id})
msg.update({'_reply_q': self._get_reply_q()})
rpc_common._add_unique_id(msg)
unique_id = msg[rpc_common.UNIQUE_ID]
rpc_common.pack_context(msg, context)
msg = rpc_common.serialize_msg(msg)
if wait_for_reply:
self._waiter.listen(msg_id)
log_msg = "CALL msg_id: %s " % msg_id
else:
log_msg = "CAST unique_id: %s " % unique_id
try:
with self._get_connection(rpc_common.PURPOSE_SEND) as conn:
if notify:
exchange = self._get_exchange(target)
log_msg += "NOTIFY exchange '%(exchange)s'" \
" topic '%(topic)s'" % {
'exchange': exchange,
'topic': target.topic}
LOG.debug(log_msg)
conn.notify_send(exchange, target.topic, msg, retry=retry)
elif target.fanout:
log_msg += "FANOUT topic '%(topic)s'" % {
'topic': target.topic
}
LOG.debug(log_msg)
conn.fanout_send(target.fanout, msg, retry=retry)
else:
topic = target.topic
exchange = self._get_exchange(target)
if target.server:
topic = '%s.%s' % (target.topic, target.server)
log_msg += "exchange '%(exchange)s' topic '%(topic)s'" % \
{'exchange': exchange, 'topic': target.topic}
LOG.debug(log_msg)
conn.topic_send(exchange_name=exchange,
topic=topic,
msg=msg,
timeout=timeout,
retry=retry)
if wait_for_reply:
result = self._waiter.wait(msg_id, timeout)
if isinstance(result, Exception):
raise result
return result
finally:
if wait_for_reply:
self._waiter.unlisten(msg_id)
def rpc_asyncrequest(self, ctxt,
asyncrequest, rpc_target, rpc_method,
rpc_ctxt, rpc_args):
"""async respone check"""
session = get_session()
finishtime = ctxt.get('finishtime', None)
asyncrequest = AsyncRequest(**asyncrequest)
pre_run = ctxt.pop('pre_run', None)
after_run = ctxt.pop('after_run', None)
post_run = ctxt.pop('post_run', None)
if finishtime and int(realnow()) >= finishtime:
asyncrequest.resultcode = manager_common.RESULT_OVER_FINISHTIME
asyncrequest.result = 'Async request over finish time'
asyncrequest.status = manager_common.FINISH
try:
session.add(asyncrequest)
session.flush()
except DBDuplicateEntry:
LOG.warning('Async request record DBDuplicateEntry')
except DBError as e:
LOG.error('Async request record DBError %s: %s' % (e.__class__.__name__, e.message))
return
if not self.is_active:
asyncrequest.resultcode = manager_common.SCHEDULER_STATUS_ERROR
asyncrequest.result = 'Rpc server not active now'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
try:
if pre_run:
pre_run = self._compile('pre', pre_run)
if after_run:
after_run = self._compile('after', after_run)
if post_run:
post_run = self._compile('post', post_run)
except (KeyError, jsonutils.ValidationError):
asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR
asyncrequest.result = 'Rpc server can not find executer or ctxt error'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
# except Exception:
# LOG.exception('wtf')
# raise
if rpc_ctxt.get('agents') is None:
wait_agents = [x[0] for x in model_query(session, Agent.agent_id,
filter=Agent.status > manager_common.DELETED).all()]
else:
wait_agents = rpc_ctxt.get('agents')
rpc_ctxt.update({'request_id': asyncrequest.request_id,
'expire': asyncrequest.expire,
'finishtime': asyncrequest.finishtime})
try:
target = Target(**rpc_target)
rpc = get_client()
except Exception:
LOG.error('Prepare rpc clinet error')
asyncrequest.resultcode = manager_common.SCHEDULER_PREPARE_ERROR
asyncrequest.result = 'Rpc server prepare rpc clinet error'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
if pre_run:
try:
pre_run.run(asyncrequest, wait_agents)
except RpcServerCtxtException as e:
asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR
asyncrequest.result = 'Rpc server ctxt pre function fail: %s' % e.message
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
session.add(asyncrequest)
session.flush()
LOG.debug('Try cast rpc method %s' % rpc_method)
try:
rpc.cast(target, ctxt=rpc_ctxt, msg={'method': rpc_method, 'args': rpc_args})
except AMQPDestinationNotFound:
asyncrequest.resultcode = manager_common.SEND_FAIL
asyncrequest.result = 'Async %s request send fail, AMQPDestinationNotFound' % rpc_method
asyncrequest.status = manager_common.FINISH
session.flush()
return
LOG.debug('Cast %s to %s success' % (asyncrequest.request_id, target.to_dict()))
if after_run:
try:
after_run.run(asyncrequest, wait_agents)
except RpcServerCtxtException as e:
asyncrequest.result = 'Async request %s cast success, ' \
'ctxt after function error~%s' % (rpc_method, e.message)
else:
asyncrequest.result = 'Async request %s cast success' % rpc_method
finally:
session.flush()
request_id = asyncrequest.request_id
finishtime = asyncrequest.finishtime
deadline = asyncrequest.deadline + 1
expire = asyncrequest.expire
if expire:
storage = get_cache()
else:
storage = session
def check_respone():
wait = finishtime - int(time.time())
# 先等待3秒,可以做一次提前检查
if wait > 3:
eventlet.sleep(3)
no_response_agents = set(wait_agents)
interval = int(wait / 10)
if interval < 3:
interval = 3
elif interval > 10:
interval = 10
not_overtime = 2
while True:
no_response_agents = responeutils.norespones(storage=storage,
request_id=request_id,
agents=no_response_agents)
if not no_response_agents:
break
if int(time.time()) < finishtime:
eventlet.sleep(interval)
if int(time.time()) > deadline:
not_overtime -= 1
if not not_overtime:
break
eventlet.sleep(1)
LOG.debug('Not response agents count %d' % len(no_response_agents))
bulk_data = []
agent_time = int(time.time())
for agent_id in no_response_agents:
data = dict(request_id=request_id,
agent_id=agent_id,
agent_time=agent_time,
server_time=agent_time,
resultcode=manager_common.RESULT_OVER_FINISHTIME,
result='Agent respone overtime')
bulk_data.append(data)
responeutils.bluk_insert(storage, no_response_agents, bulk_data, expire)
asyncrequest.status = manager_common.FINISH
if no_response_agents:
asyncrequest.resultcode = manager_common.RESULT_NOT_ALL_SUCCESS
asyncrequest.result = 'agents not respone, count:%d' % len(no_response_agents)
else:
asyncrequest.resultcode = manager_common.RESULT_SUCCESS
asyncrequest.result = 'all agent respone result'
session.flush()
if post_run:
try:
post_run.run(asyncrequest, no_response_agents)
except RpcServerCtxtException as e:
asyncrequest.result += (' ctxt post function error~%s' % e.message)
session.flush()
session.close()
threadpool.add_thread(safe_func_wrapper, check_respone, LOG)