def opentime(self, req, group_id, objtype, entity, body=None): """修改开服时间接口""" body = body or {} group_id = int(group_id) entity = int(entity) if objtype != common.GAMESERVER: raise InvalidArgument('Api just for %s' % common.GAMESERVER) opentime = int(body.pop('opentime')) if opentime < 0 or opentime >= int(time.time()) + 86400 * 15: raise InvalidArgument('opentime value error') session = endpoint_session() with session.begin(): query = model_query(session, AppEntity, filter=AppEntity.entity == entity) _entity = query.one() if _entity.objtype != objtype: raise InvalidArgument('Entity is not %s' % objtype) if _entity.group_id != group_id: raise InvalidArgument('Entity group %d not match %d' % (_entity.group_id, group_id)) metadata, ports = self._entityinfo(req=req, entity=entity) target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc = get_client() finishtime, timeout = rpcfinishtime() # with session.begin(): rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime}, msg={'method': 'opentime_entity', 'args': dict(entity=entity, opentime=opentime)}, timeout=timeout) query.update({'opentime': opentime}) if not rpc_ret: raise RpcResultError('change entity opentime result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('change entity opentime fail %s' % rpc_ret.get('result')) return resultutils.results(result='change entity %d opentime success' % entity)
def _start_database(self, database, **kwargs): req = kwargs.pop('req') entity = int(database.reflection_id) _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'start_entity', 'args': dict(entity=entity) }, timeout=timeout) if not rpc_ret: raise RpcResultError('create entitys result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('create entity fail %s' % rpc_ret.get('result')) return rpc_ret
def _revoke_database_user(self, database, auth, **kwargs): req = kwargs.pop('req') entity = int(database.reflection_id) _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] if not metadata: raise InvalidArgument('Traget database agent is offline') target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'revoke_entity', 'args': dict(entity=entity, auth=auth) }, timeout=timeout) if not rpc_ret: raise RpcResultError('revoke grant from database result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('revoke grant from database fail %s' % rpc_ret.get('result')) return rpc_ret
def wapper(): eventlet.sleep(random.randint(0, 5)) # save report log session = get_session() report = AgentReportLog(**snapshot) session.add(report) session.flush() session.close() process = snapshot.get('running') + snapshot.get('sleeping') free = snapshot.get('free') + snapshot.get('cached') conns = snapshot.get('syn') + snapshot.get('enable') cputime = snapshot.get('iowait') + snapshot.get('user') \ + snapshot.get('system') + snapshot.get('nice')\ + snapshot.get('irq') + snapshot.get('sirq') rpc = get_client() # send to rpc server rpc.cast(targetutils.target_rpcserver(fanout=True), ctxt = {}, msg={'method': 'changesource', 'args': {'agent_id': agent_id, 'free': free, 'process': process, 'cputime': cputime, 'iowait': snapshot.get('iowait'), 'left': snapshot.get('left'), 'fds': snapshot.get('num_fds'), 'conns': conns, 'metadata': metadata, }})
def wapper(): if self.delay: eventlet.sleep(self.delay) data = self.notify[keyword] target = Target(**data.pop('target')) func = getattr(get_client(), data.pop('method')) func(target, **data)
def create(self, req, body=None): body = body or {} jsonutils.schema_validate(body, SCHEDULEJOBSCHEMA) start = datetime.datetime.fromtimestamp(body['start']) if start < int(time.time()) + 300: raise InvalidArgument('Do not add a scheduler in 5 min') job_id = uuidutils.Gkey() rpc = get_client() glock = get_global().lock('autorelase') with glock(targetutils.schedule_job(), 30): job_result = rpc.call(targetutils.target_anyone( manager_common.SCHEDULER), ctxt={'finishtime': ""}, msg={ 'method': 'scheduler', 'args': { 'job_id': job_id, 'jobdata': body } }) # job interval if not job_result: raise RpcResultError('delete_agent_precommit result is None') if job_result.get('resultcode') != manager_common.RESULT_SUCCESS: return resultutils.results(result=job_result.get('result')) return resultutils.results( result='Create scheduler job:%d success' % job_id)
def notify_create(target, agent_id, entity, body, action='create'): rpc = get_client() body.setdefault('entity', entity) finishtime = body.pop('finishtime', None) if finishtime is None: finishtime, timeout = rpcfinishtime() else: timeout = finishtime - int(time.time()) + 1 if timeout < 3: raise InvalidArgument('Timeout less then 3') create_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ], 'entitys': [ entity, ] }, msg={ 'method': '%s_entity' % action, 'args': body }, timeout=timeout) return create_ret
def delete(self, req, agent_id, body=None): """call buy agent""" # if force is true # will not notify agent, just delete agent from database body = body or {} force = body.get('force', False) rpc = get_client() global_data = get_global() metadata = None with global_data.delete_agent(agent_id) as agent: if not force: metadata = BaseContorller.agent_metadata(agent.agent_id) if metadata is None: raise RpcPrepareError('Can not delete offline agent, try force') agent_ipaddr = metadata.get('local_ip') secret = uuidutils.generate_uuid() # tell agent wait delete finishtime, timeout = rpcfinishtime() delete_agent_precommit = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': finishtime}, msg={'method': 'delete_agent_precommit', 'args': {'agent_id': agent.agent_id, 'agent_type': agent.agent_type, 'host': agent.host, 'agent_ipaddr': agent_ipaddr, 'secret': secret} }, timeout=timeout) if not delete_agent_precommit: raise RpcResultError('delete_agent_precommit result is None') if delete_agent_precommit.get('resultcode') != manager_common.RESULT_SUCCESS: return resultutils.results(result=delete_agent_precommit.get('result'), resultcode=manager_common.RESULT_ERROR) # if not force: # tell agent delete itself finishtime = rpcfinishtime()[0] LOG.info('Delete agent %s postcommit with secret %s' % (agent_ipaddr, secret)) rpc.cast(targetutils.target_agent(agent), ctxt={'finishtime': finishtime}, msg={'method': 'delete_agent_postcommit', 'args': {'agent_id': agent.agent_id, 'agent_type': agent.agent_type, 'host': agent.host, 'agent_ipaddr': agent_ipaddr, 'secret': secret}}) def wapper(): rpc.cast(targetutils.target_rpcserver(fanout=True), msg={'method': 'deletesource', 'args': {'agent_id': agent_id}}) threadpool.add_thread(safe_func_wrapper, wapper, LOG) result = resultutils.results(result='Delete agent success', data=[dict(agent_id=agent.agent_id, host=agent.host, status=agent.status, metadata=metadata, ports_range=jsonutils.safe_loads_as_bytes(agent.ports_range) or []) ]) return result
def _expire(): try: BaseContorller._agent_metadata_expire(agent_id, expire + fix) except AgentMetadataMiss: # 元数据丢失, 通知agent重新上报 rpc = get_client() # send to rpc server rpc.cast(targetutils.target_all(fanout=True), ctxt={'agents': [agent_id, ]}, msg={'method': 'flush_metadata', 'args': {'expire': expire}})
def _unbond_database(self, session, master, slave, relation, **kwargs): req = kwargs.pop('req') entity = int(slave.reflection_id) with session.begin(subtransactions=True): _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] if not metadata: raise InvalidArgument('Traget database agent is offline') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() # 发送master信息到从库所在agent rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime + 3, 'agents': [ agent_id, ] }, msg={ 'method': 'unbond_entity', 'args': dict(entity=entity, force=kwargs.get('force', False), master=dict( database_id=master.database_id, ready=relation.ready, schemas=[ schema.schema for schema in master.schemas ], )) }, timeout=timeout + 3) if not rpc_ret: raise RpcResultError('unbond database result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('unbond database fail %s' % rpc_ret.get('result')) # 绑定状态设置就绪 session.delete(relation) session.flush() auth = privilegeutils.mysql_replprivileges( slave.database_id, metadata.get('local_ip')) auth['schema'] = '*' return self._revoke_database_user(master, auth, req=req)
def _slave_database(self, session, master, slave, **kwargs): req = kwargs.pop('req') with session.begin(subtransactions=True): # get slave host and port _host, _port = self._get_entity(req=req, entity=int(slave.reflection_id), raise_error=True) entity = int(master.reflection_id) _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] if not metadata: raise InvalidArgument('Traget database agent is offline') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() # 发送slave信息到主库所在agent rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime + 5, 'agents': [ agent_id, ] }, msg={ 'method': 'slave_entity', 'args': dict(entity=entity, schemas=kwargs.get('schemas'), file=kwargs.get('file'), position=kwargs.get('position'), bond=dict( database_id=slave.database_id, host=_host, port=_port)) }, timeout=timeout + 5) if not rpc_ret: raise RpcResultError( 'bond slave for master database result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('bond slave for master database fail %s' % rpc_ret.get('result')) return rpc_ret
def send_asyncrequest(asyncrequest, rpc_target, rpc_ctxt, rpc_method, rpc_args=None, async_ctxt=None): rpc = get_client() session = get_session() try: rpc.cast( targetutils.target_rpcserver(), # ctxt={'finishtime': asyncrequest.finishtime-2}, ctxt=async_ctxt or {}, msg={ 'method': 'asyncrequest', 'args': { 'asyncrequest': asyncrequest.to_dict(), 'rpc_target': rpc_target.to_dict(), 'rpc_method': rpc_method, 'rpc_ctxt': rpc_ctxt, 'rpc_args': rpc_args or dict() } }) except AMQPDestinationNotFound as e: LOG.error('Send async request to scheduler fail %s' % e.__class__.__name__) asyncrequest.status = manager_common.FINISH asyncrequest.result = e.message asyncrequest.resultcode = manager_common.SCHEDULER_NOTIFY_ERROR try: session.add(asyncrequest) session.flush() except DBDuplicateEntry: LOG.warning( 'Async request rpc call result is None, but recode found') except Exception as e: if LOG.isEnabledFor(logging.DEBUG): LOG.exception('Async request rpc cast fail') else: LOG.error('Async request rpc cast unkonw error') asyncrequest.status = manager_common.FINISH asyncrequest.result = 'Async request rpc cast error: %s' % e.__class__.__name__ asyncrequest.resultcode = manager_common.RESULT_ERROR try: session.add(asyncrequest) session.flush() raise except DBDuplicateEntry: LOG.warning( 'Async request rpc call result is None, but recode found')
def _ready_relation(self, session, master, slave, relation, **kwargs): req = kwargs.pop('req') entity = int(slave.reflection_id) with session.begin(subtransactions=True): schemas = [schema.schema for schema in master.schemas] _host, _port = self._get_entity(req, int(master.reflection_id), raise_error=True) _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] if not metadata: raise InvalidArgument('Traget database agent is offline') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() # 发送master信息到从库所在agent rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'entity_replication_ready', 'args': dict(entity=entity, master=dict( database_id=master.database_id, host=_host, port=_port, schemas=schemas)) }, timeout=timeout) if not rpc_ret: raise RpcResultError('get replication status result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('get replication status fail %s' % rpc_ret.get('result')) # 绑定状态设置就绪 relation.ready = True return rpc_ret
def _bond_database(self, session, master, slave, relation, **kwargs): req = kwargs.pop('req') entity = int(slave.reflection_id) with session.begin(subtransactions=True): _entity = entity_controller.show(req=req, entity=entity, endpoint=common.DB, body={'ports': False})['data'][0] agent_id = _entity['agent_id'] metadata = _entity['metadata'] if not metadata: raise InvalidArgument('Traget database agent is offline') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.DB rpc = get_client() finishtime, timeout = rpcfinishtime() # 发送master信息到从库所在agent rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime + 5, 'agents': [ agent_id, ] }, msg={ 'method': 'bond_entity', 'args': dict(entity=entity, force=kwargs.get('force', False), master=dict( database_id=master.database_id, host=kwargs.get('host'), port=kwargs.get('port'), passwd=kwargs.get('passwd'), file=kwargs.get('file'), position=kwargs.get('position'), schemas=kwargs.get('schemas'))) }, timeout=timeout + 5) if not rpc_ret: raise RpcResultError('bond database result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('bond database fail %s' % rpc_ret.get('result')) # 绑定状态设置就绪 relation.ready = True if not kwargs.get('schemas') else False return rpc_ret
def _check_file(agent_id, objtype, appfile): metadata = BaseContorller.agent_metadata(agent_id) if not metadata: return False target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) rpc = get_client() rpc_ret = rpc.call(target, ctxt={'agents': [agent_id, ]}, msg={'method': 'check_file', 'args': dict(objtype=objtype, appfile=appfile)}) if not rpc_ret: LOG.error('Rpc call result is None') return False if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: return False return True
def continues(self, req, uuid, body=None): """中途失败的合服任务再次运行""" session = endpoint_session() query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid) query = query.options(joinedload(MergeTask.entitys, innerjoin=False)) etask = query.one() if etask.status == common.MERGEFINISH: raise InvalidArgument('Merge task has all ready finished') _query = model_query(session, AppEntity, filter=AppEntity.entity == etask.entity) _query = _query.options( joinedload(AppEntity.databases, innerjoin=False)) appentity = _query.one_or_none() if not appentity or not appentity.databases or appentity.objtype != common.GAMESERVER: LOG.error('Etask entity can not be found or type/database error') raise exceptions.MergeException( 'Etask entity can not be found or type/database error') databases = self._database_to_dict(appentity) rpc = get_client() metadata, ports = self._entityinfo(req=req, entity=appentity.entity) target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc_ret = rpc.call(target, ctxt={'agents': [ appentity.agent_id, ]}, msg={ 'method': 'continue_merge', 'args': dict(entity=etask.entity, uuid=uuid, databases=databases) }) if not rpc_ret: raise RpcResultError('continue entity result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('continue entity fail %s' % rpc_ret.get('result')) return resultutils.results( result='continue merge task command has been send', data=[dict(uuid=etask.uuid, entity=etask.entity)])
def chioces(endpoint, includes=None, weighters=None): """return a agents list sort by weigher""" rpc = get_client() chioces_result = rpc.call(targetutils.target_rpcserver(), ctxt=dict(), msg={ 'method': 'chioces', 'args': { 'target': endpoint, 'includes': includes, 'weighters': weighters } }) if not chioces_result: raise RpcResultError('Active agent chioces result is None') if chioces_result.pop('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('Call agent chioces fail: ' + chioces_result.get('result')) return chioces_result['agents']
def notify_delete(target, agent_id, entity, body, action='delete'): rpc = get_client() body.setdefault('entity', entity) token = body.get('token') finishtime = body.pop('finishtime', None) if finishtime is None: finishtime, timeout = rpcfinishtime() else: timeout = finishtime - int(time.time()) + 1 if timeout < 3: raise InvalidArgument('Timeout less then 3') if token: # send a delete token rpc.cast(target, ctxt={ 'finishtime': finishtime, 'entitys': [ entity, ] }, msg={ 'method': 'entity_token', 'args': { 'entity': entity, 'token': token } }) delete_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ], 'entitys': [ entity, ] }, msg={ 'method': '%s_entity' % action, 'args': body }, timeout=timeout) return delete_ret
def readlog(self, req, agent_id, body=None): """call by agent""" body = body or {} lines = body.get('lines', 10) rpc = get_client() metadata = BaseContorller.agent_metadata(agent_id) if metadata is None: raise RpcPrepareError('Can not get log from offline agent: %d' % agent_id) session = get_session() query = model_query(session, Agent, filter=Agent.agent_id == agent_id) agent = query.one() rpc_ret = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': rpcfinishtime()[0]}, msg={'method': 'readlog', 'args': {'lines': lines}}) if not rpc_ret: raise RpcResultError('Get log agent rpc result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('Get log agent rpc result: ' + rpc_ret.get('result')) return resultutils.results(result=rpc_ret.get('result'), data=[rpc_ret.get('uri')])
def readlog(self, req, endpoint, entity, body=None): body = body or {} path = body.get('path') lines = body.get('lines', 10) if not path or '..' in path: raise InvalidArgument('Path value error') endpoint = validateutils.validate_endpoint(endpoint) entity = int(entity) session = get_session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) _entity = query.one_or_none() if not _entity: raise InvalidArgument('no entity found for %s' % endpoint) metadata = BaseContorller.agent_metadata(_entity.agent_id) if not metadata: raise InvalidArgument('Can not get log from off line agent') target = targetutils.target_agent_by_string(manager_common.APPLICATION, metadata.get('host')) target.namespace = endpoint rpc = get_client() rpc_ret = rpc.call(target, ctxt={'finishtime': rpcfinishtime()[0]}, msg={ 'method': 'readlog', 'args': { 'entity': entity, 'path': path, 'lines': lines } }) if not rpc_ret: raise RpcResultError('Get %s.%d log rpc result is None' % (endpoint, entity)) if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise InvalidArgument(('Get %s.%d log agent rpc result: ' % (endpoint, entity)) + rpc_ret.get('result')) return resultutils.results(result=rpc_ret.get('result'), data=[rpc_ret.get('uri')])
def active(self, req, agent_id, body=None): """call buy client""" body = body or {} status = body.get('status', manager_common.ACTIVE) if status not in (manager_common.ACTIVE, manager_common.UNACTIVE): raise InvalidArgument('Argument status not right') rpc = get_client() session = get_session() query = model_query(session, Agent, filter=and_(Agent.agent_id == agent_id, Agent.status > manager_common.DELETED)) agent = query.one() # make sure agent is online metadata = BaseContorller.agent_metadata(agent.agent_id) if metadata is None: raise RpcPrepareError('Can not active or unactive a offline agent: %d' % agent_id) agent_ipaddr = metadata.get('local_ip') with session.begin(): agent.update({'status': status}) active_agent = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': rpcfinishtime()[0]}, msg={'method': 'active_agent', 'args': {'agent_id': agent_id, 'agent_ipaddr': agent_ipaddr, 'status': status} }) if not active_agent: raise RpcResultError('Active agent rpc result is None') if active_agent.pop('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('Call agent active or unactive fail: ' + active_agent.get('result')) result = resultutils.results(result=active_agent.pop('result'), data=[dict(agent_id=agent.agent_id, host=agent.host, agent_type=agent.agent_type, metadata=metadata, status=agent.status) ]) return result
def post_create_entity(self, entity, endpoint, **kwargs): entity = int(entity) endpoint = validateutils.validate_endpoint(endpoint) session = get_session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) _entity = query.one() metadata = BaseContorller.agent_metadata(_entity.agent_id) if not metadata: raise RpcPrepareError('Agent not online, can not sen post create') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host'), ) target.namespace = endpoint body = dict(entity=entity) body.update(kwargs) rpc = get_client() rpc.cast(target, ctxt={ 'finishtime': body.pop('finishtime', rpcfinishtime()[0]), 'entitys': [ entity, ] }, msg={ 'method': 'post_create_entity', 'args': body }) return resultutils.results(result='notify post create success', data=[ dict(entity=entity, agent_id=_entity.agent_id, endpoint=endpoint) ])
def swallow(self, req, entity, body=None): """合服内部接口,一般由agent调用 用于新实体吞噬旧实体的区服和数据库""" body = body or {} entity = int(entity) uuid = body.get('uuid') if not uuid: raise InvalidArgument('Merger uuid is None') session = endpoint_session() query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid) query = query.options(joinedload(MergeTask.entitys, innerjoin=False)) glock = get_gamelock() rpc = get_client() with session.begin(): etask = query.one_or_none() if not etask: raise InvalidArgument('Not task exit with %s' % uuid) # 新实体不匹配 if etask.entity != body.get('entity'): raise InvalidArgument('New entity not %d' % etask.entity) # 找到目标实体 appentity = None for _entity in etask.entitys: if _entity.entity == entity: if _entity.status != common.MERGEING: if _entity.status != common.SWALLOWING: raise InvalidArgument( 'Swallow entity find status error') if not _entity.databases or not _entity.areas: raise InvalidArgument( 'Entity is swallowing but database or ares is None' ) LOG.warning('Entit is swallowing, return saved data') return resultutils.results( result='swallow entity is success', data=[ dict(databases=jsonutils.loads_as_bytes( _entity.databases), areas=jsonutils.loads_as_bytes( _entity.areas)) ]) _query = model_query(session, AppEntity, filter=AppEntity.entity == entity) _query = _query.options( joinedload(AppEntity.databases, innerjoin=False)) appentity = _query.one_or_none() break if not appentity: raise InvalidArgument('Can not find app entity?') if appentity.objtype != common.GAMESERVER: raise InvalidArgument('objtype error, entity not %s' % common.GAMESERVER) if appentity.status != common.MERGEING: raise InvalidArgument('find status error, when swallowing') databases = self._database_to_dict(appentity) areas = [area.to_dict() for area in appentity.areas] if not databases or not areas: LOG.error('Entity no areas or databases record') return resultutils.results( result='swallow entity fail, ' 'target entity can not found database or areas', resultcode=manager_common.RESULT_ERROR) with glock.grouplock(group=appentity.group_id): # 发送吞噬命令到目标区服agent metadata, ports = self._entityinfo(req=req, entity=entity) target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc_ret = rpc.call(target, ctxt={'agents': [ appentity.agent_id, ]}, msg={ 'method': 'swallow_entity', 'args': dict(entity=entity) }) if not rpc_ret: raise RpcResultError('swallow entity result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('swallow entity fail %s' % rpc_ret.get('result')) # 修改实体在合服任务中的状态,存储areas以及databases appentity.status = common.SWALLOWING _entity.status = common.SWALLOWING _entity.areas = jsonutils.dumps(areas) _entity.databases = jsonutils.dumps(databases) session.flush() return resultutils.results( result='swallow entity is success', data=[dict(databases=databases, areas=areas)])
def swallowed(self, req, entity, body=None): """ 合服内部接口,一般由agent调用 用于新实体吞噬旧实体的区服完成后调用 调用后将设置appentity为deleted状态 """ body = body or {} entity = int(entity) uuid = body.get('uuid') if not uuid: raise InvalidArgument('Merger uuid is None') session = endpoint_session() query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid) query = query.options(joinedload(MergeTask.entitys, innerjoin=False)) glock = get_gamelock() rpc = get_client() appentity = None with session.begin(): etask = query.one_or_none() if not etask: raise InvalidArgument('Not task exit with %s' % uuid) # 新实体不匹配 if etask.entity != body.get('entity'): raise InvalidArgument('New entity not %d' % etask.entity) for _entity in etask.entitys: if _entity.entity == entity: if _entity.status != common.SWALLOWING: raise InvalidArgument( 'Swallowed entity find status error') _query = model_query(session, AppEntity, filter=AppEntity.entity == entity) _query = _query.options( joinedload(AppEntity.databases, innerjoin=False)) appentity = _query.one_or_none() break if not appentity: raise InvalidArgument('Can not find app entity?') if appentity.objtype != common.GAMESERVER: raise InvalidArgument('objtype error, entity not %s' % common.GAMESERVER) if appentity.status != common.SWALLOWING: raise InvalidArgument('find status error, when swallowed') with glock.grouplock(group=appentity.group_id): # 发送吞噬完成命令到目标区服agent metadata, ports = self._entityinfo(req=req, entity=entity) target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc_ret = rpc.call(target, ctxt={'agents': [ appentity.agent_id, ]}, msg={ 'method': 'swallowed_entity', 'args': dict(entity=entity) }) if not rpc_ret: raise RpcResultError('swallowed entity result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('swallowed entity fail %s' % rpc_ret.get('result')) # appentity状态修改为deleted appentity.status = common.DELETED # 修改实体在合服任务中的状态 _entity.status = common.MERGEED session.flush() # area绑定新实体 _query = model_query(session, GameArea, filter=GameArea.entity == entity) _query.update({'entity': etask.entity}) session.flush() def _unquote(): LOG.info('Swallowed %d finish, try unquote database' % appentity.entity) for database in appentity.databases: try: schema_controller.unquote(req, quote_id=database.quote_id) except Exception: LOG.error('Delete database quote fail') eventlet.spawn_n(_unquote) return resultutils.results( result='swallowed entity is success', data=[ dict(databases=jsonutils.loads_as_bytes(_entity.databases), areas=jsonutils.loads_as_bytes(_entity.areas)) ])
def clean(self, req, group_id, objtype, entity, body=None): """彻底删除entity""" body = body or {} action = body.pop('clean', 'unquote') force = False ignores = body.pop('ignores', []) if action not in ('delete', 'unquote', 'force'): raise InvalidArgument('clean option value error') if action == 'force': action = 'delete' force = True group_id = int(group_id) entity = int(entity) session = endpoint_session() glock = get_gamelock() metadata, ports = self._entityinfo(req=req, entity=entity) if not metadata: raise InvalidArgument('Agent offline, can not delete entity') query = model_query(session, AppEntity, filter=AppEntity.entity == entity) query = query.options(joinedload(AppEntity.databases, innerjoin=False)) _entity = query.one() rollbacks = [] def _rollback(): for back in rollbacks: __database_id = back.get('database_id') __schema = back.get('schema') __quote_id = back.get('quote_id') rbody = dict(quote_id=__quote_id, entity=entity) rbody.setdefault(dbcommon.ENDPOINTKEY, common.NAME) try: schema_controller.bond(req, database_id=__database_id, schema=__schema, body=rbody) except Exception: LOG.error('rollback entity %d quote %d.%s.%d fail' % (entity, __database_id, schema, __quote_id)) with glock.grouplock(group=group_id): target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc = get_client() finishtime, timeout = rpcfinishtime() LOG.warning('Clean entity %s.%d with action %s' % (objtype, entity, action)) with session.begin(): rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime}, msg={'method': 'stoped', 'args': dict(entity=entity)}) if not rpc_ret: raise RpcResultError('check entity is stoped result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('check entity is stoped fail, running') with session.begin(): if _entity.status != common.DELETED: raise InvalidArgument('Entity status is not DELETED, ' 'mark status to DELETED before delete it') if _entity.objtype != objtype: raise InvalidArgument('Objtype not match') if _entity.group_id != group_id: raise InvalidArgument('Group id not match') # esure database delete if action == 'delete': LOG.warning('Clean option is delete, can not rollback when fail') if not force: for _database in _entity.databases: schema = '%s_%s_%s_%d' % (common.NAME, objtype, _database.subtype, entity) schema_info = schema_controller.show(req=req, database_id=_database.database_id, schema=schema, body={'quotes': True})['data'][0] quotes = {} for _quote in schema_info['quotes']: quotes[_quote.get('quote_id')] = _quote.get('desc') if _database.quote_id not in quotes.keys(): # if set(quotes) != set([_database.quote_id]): result = 'delete %s:%d fail' % (objtype, entity) reason = ': database [%d].%s quote: %s' % (_database.database_id, schema, str(quotes)) return resultutils.results(result=(result + reason), resultcode=manager_common.RESULT_ERROR) quotes.pop(_database.quote_id) for quote_id in quotes.keys(): if quotes[quote_id] in ignores: quotes.pop(quote_id, None) if quotes: if LOG.isEnabledFor(logging.DEBUG): LOG.debug('quotes not match for %d: %s' % (schema_info['schema_id'], schema)) for quote_id in quotes.keys(): LOG.debug('quote %d: %s exist' % (quote_id, quotes[quote_id])) LOG.debug('Can not delete schema before delete quotes') return resultutils.results(result='Quotes not match', resultcode=manager_common.RESULT_ERROR) LOG.info('Databae quotes check success for %s' % schema) # clean database for _database in _entity.databases: schema = '%s_%s_%s_%d' % (common.NAME, objtype, _database.subtype, entity) if action == 'delete': LOG.warning('Delete schema %s from %d' % (schema, _database.database_id)) try: schema_controller.delete(req=req, database_id=_database.database_id, schema=schema, body={'unquotes': [_database.quote_id], 'ignores': ignores, 'force': force}) except GopdbError as e: LOG.error('Delete schema:%s from %d fail, %s' % (schema, _database.database_id, e.message)) if not force: raise e except Exception: LOG.exception('Delete schema:%s from %d fail' % (schema, _database.database_id)) if not force: raise elif action == 'unquote': LOG.info('Try unquote %d' % _database.quote_id) try: quote = schema_controller.unquote(req=req, quote_id=_database.quote_id)['data'][0] if quote.get('database_id') != _database.database_id: LOG.critical('quote %d with database %d, not %d' % (_database.quote_id, quote.get('database_id'), _database.database_id)) raise RuntimeError('Data error, quote database not the same') rollbacks.append(dict(database_id=_database.database_id, quote_id=_database.quote_id, schema=schema)) except Exception as e: LOG.error('Unquote %d fail, try rollback' % _database.quote_id) if not force: threadpool.add_thread(_rollback) raise e token = uuidutils.generate_uuid() LOG.info('Send delete command with token %s' % token) session.delete(_entity) session.flush() try: entity_controller.delete(req, common.NAME, entity=entity, body=dict(token=token)) except Exception as e: # roll back unquote threadpool.add_thread(_rollback) raise e return resultutils.results(result='delete %s:%d success' % (objtype, entity), data=[dict(entity=entity, objtype=objtype, ports=ports, metadata=metadata)])
def delete(self, req, group_id, objtype, entity, body=None): """标记删除entity""" body = body or {} force = body.get('force', False) group_id = int(group_id) entity = int(entity) session = endpoint_session() glock = get_gamelock() metadata, ports = self._entityinfo(req=req, entity=entity) if not metadata: raise InvalidArgument('Agent offline, can not delete entity') query = model_query(session, AppEntity, filter=AppEntity.entity == entity) if objtype == common.GAMESERVER: query = query.options(joinedload(AppEntity.areas, innerjoin=False)) _entity = query.one() if _entity.status == common.DELETED: return resultutils.results(result='mark %s entity delete success' % objtype, data=[ dict(entity=entity, objtype=objtype, ports=ports, metadata=metadata) ]) if _entity.objtype != objtype: raise InvalidArgument('Objtype not match') if _entity.group_id != group_id: raise InvalidArgument('Group id not match') target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc = get_client() with glock.grouplock(group=group_id): if objtype == common.GMSERVER: if model_count_with_key( session, AppEntity, filter=AppEntity.group_id == group_id) > 1: raise InvalidArgument( 'You must delete other objtype entity before delete gm' ) if model_count_with_key( session, Package, filter=Package.group_id == group_id) > 1: raise InvalidArgument( 'You must delete other Package before delete gm') elif objtype == common.CROSSSERVER: if model_count_with_key( session, AppEntity, filter=AppEntity.cross_id == _entity.entity): raise InvalidArgument('Cross server are reflected') with session.begin(): # 确认实体没有运行 rpc_ret = rpc.call(target, ctxt={'agents': [ _entity.agent_id, ]}, msg={ 'method': 'stoped', 'args': dict(entity=entity) }) if not rpc_ret: raise RpcResultError('check entity stoped result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('check entity fail %s' % rpc_ret.get('result')) _entity.status = common.DELETED session.flush() if objtype == common.GAMESERVER: # 删除所有资源版本引用 if _entity.versions: for quote in six.itervalues( jsonutils.loads_as_bytes(_entity.versions)): threadpool.add_thread(cdnquote_controller.delete, req, quote.get('quote_id')) _entity.versions = None session.flush() if _entity.areas: if len(_entity.areas) > 1: raise InvalidArgument('%s areas more then one' % objtype) area = _entity.areas[0] if not force: if _entity.entity != model_max_with_key( session, AppEntity.entity, filter=and_( AppEntity.objtype == common.GAMESERVER, AppEntity.group_id == group_id)): raise InvalidArgument( 'entity %d is not the last gamesvr entity in group' % entity) session.flush() session.delete(area) session.flush() _query = model_query( session, PackageArea, filter=PackageArea.area_id == area.area_id) _query.delete() session.flush() rpc.cast(target, ctxt={'agents': [ _entity.agent_id, ]}, msg={ 'method': 'change_status', 'args': dict(entity=entity, status=common.DELETED) }) return resultutils.results(result='mark %s entity delete success' % objtype, data=[ dict(entity=entity, objtype=objtype, ports=ports, metadata=metadata) ])
def reset(self, req, group_id, objtype, entity, body=None): """重置Entity程序以及配置""" body = body or {} group_id = int(group_id) entity = int(entity) # 重置程序文件,为空表示不需要重置程序文件 appfile = body.pop(common.APPFILE, None) # 重置数据库信息 databases = body.pop('databases', False) # 重置主服务器信息(gameserver专用) chiefs = body.pop('chiefs', False) # 查询entity信息 session = endpoint_session() query = model_query(session, AppEntity, filter=AppEntity.entity == entity) query = query.options(joinedload(AppEntity.databases, innerjoin=False)) _entity = query.one() if _entity.objtype != objtype: raise InvalidArgument('Entity is not %s' % objtype) if _entity.group_id != group_id: raise InvalidArgument('Entity group %d not match %d' % (_entity.group_id, group_id)) entityinfo = entity_controller.show(req=req, entity=entity, endpoint=common.NAME, body={'ports': False})['data'][0] agent_id = entityinfo['agent_id'] metadata = entityinfo['metadata'] if not metadata: raise InvalidArgument('Agent is off line, can not reset entity') # 需要更新数据库 if databases: miss = [] databases = {} # 从本地查询数据库信息 for database in _entity.databases: subtype = database.subtype schema = '%s_%s_%s_%d' % (common.NAME, objtype, subtype, entity) databases[subtype] = dict(host=database.host, port=database.port, user=database.user, passwd=database.passwd, schema=schema, character_set=database.character_set) # 必要数据库信息 NEEDED = common.DBAFFINITYS[objtype].keys() # 数据库信息不匹配,从gopdb接口反查数据库信息 if set(NEEDED) != set(databases.keys()): LOG.warning('Database not match, try find schema info from gopdb') quotes = schema_controller.quotes(req, body=dict(entitys=[entity, ], endpoint=common.NAME))['data'] for subtype in NEEDED: if subtype not in databases: # 从gopdb接口查询引用信息 schema = '%s_%s_%s_%d' % (common.NAME, objtype, subtype, entity) for quote_detail in quotes: # 确认引用不是从库且结构名称相等 if quote_detail['qdatabase_id'] == quote_detail['database_id'] \ and quote_detail['schema'] == schema: databases.setdefault(subtype, dict(host=quote_detail['host'], port=quote_detail['port'], user=quote_detail['user'], passwd=quote_detail['passwd'], schema=schema, character_set=quote_detail['character_set'])) miss.append(AreaDatabase(quote_id=quote_detail['quote_id'], database_id=quote_detail['qdatabase_id'], entity=entity, subtype=subtype, host=quote_detail['host'], port=quote_detail['port'], user=quote_detail['user'], passwd=quote_detail['passwd'], ro_user=quote_detail['ro_user'], ro_passwd=quote_detail['ro_passwd'], character_set=quote_detail['character_set']) ) quotes.remove(quote_detail) break if subtype not in databases: LOG.critical('Miss database of %s' % schema) # 数据库信息无法从gopdb中反查到 raise ValueError('Not %s.%s database found for %d' % (objtype, subtype, entity)) self._validate_databases(objtype, databases) # 有数据库信息遗漏 if miss: with session.begin(): for obj in miss: session.add(obj) session.flush() if objtype == common.GAMESERVER and chiefs: chiefs = {} cross_id = _entity.cross_id if cross_id is None: raise ValueError('%s.%d cross_id is None' % (objtype, entity)) query = model_query(session, AppEntity, filter=and_(AppEntity.group_id == group_id, or_(AppEntity.entity == cross_id, AppEntity.objtype == common.GMSERVER))) _chiefs = query.all() if len(_chiefs) != 2: raise ValueError('Try find %s.%d chiefs from local database error' % (objtype, entity)) for chief in _chiefs: for _objtype in (common.GMSERVER, common.CROSSSERVER): _metadata, ports = self._entityinfo(req, chief.entity) if not _metadata: raise InvalidArgument('Metadata of %s.%d is none' % (_objtype, chief.entity)) if chief.objtype == _objtype: chiefs[_objtype] = dict(entity=chief.entity, ports=ports, local_ip=_metadata.get('local_ip')) if len(chiefs) != 2: raise ValueError('%s.%d chiefs error' % (objtype, entity)) target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc = get_client() finishtime, timeout = rpcfinishtime() if appfile: finishtime += 30 timeout += 35 rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime, 'agents': [agent_id, ]}, msg={'method': 'reset_entity', 'args': dict(entity=entity, appfile=appfile, opentime=_entity.opentime, databases=databases, chiefs=chiefs)}, timeout=timeout) if not rpc_ret: raise RpcResultError('reset entity result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('reset entity fail %s' % rpc_ret.get('result')) return resultutils.results(result='reset entity %d success' % entity)
def update(self, req, group_id, objtype, entity, body=None): body = body or {} group_id = int(group_id) entity = int(entity) status = body.get('status', common.OK) if status not in (common.UNACTIVE, common.OK): raise InvalidArgument('Status not in 0, 1, 2') session = endpoint_session() glock = get_gamelock() query = model_query(session, AppEntity, filter=AppEntity.entity == entity) if objtype == common.GAMESERVER: query = query.options(joinedload(AppEntity.areas, innerjoin=False)) _entity = query.one() if status == _entity.status: return resultutils.results(result='%s entity status in same' % objtype) if _entity.status not in (common.OK, common.UNACTIVE): return resultutils.results( resultcode=manager_common.RESULT_ERROR, result='%s entity is not ok or unactive' % objtype) if _entity.objtype != objtype: raise InvalidArgument('Objtype not match') if _entity.group_id != group_id: raise InvalidArgument('Group id not match') entityinfo = entity_controller.show(req=req, entity=entity, endpoint=common.NAME, body={'ports': False})['data'][0] agent_id = entityinfo['agent_id'] metadata = entityinfo['metadata'] if not metadata: raise InvalidArgument('Agent is off line, can not reset entity') rpc = get_client() target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME if objtype == common.GAMESERVER: lock = functools.partial( glock.arealock, group=group_id, areas=[area.area_id for area in _entity.areas]) else: lock = functools.partial(glock.grouplock, group=group_id) with lock(): with session.begin(): _entity.status = status session.flush() finishtime, timeout = rpcfinishtime() rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'change_status', 'args': dict(entity=entity, status=status) }, timeout=timeout) if not rpc_ret: raise RpcResultError('change entity sttus result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('change entity status fail %s' % rpc_ret.get('result')) return resultutils.results(result='%s entity update success' % objtype)
def area(self, req, group_id, body=None): """change entity area""" body = body or {} try: group_id = int(group_id) except (TypeError, ValueError): raise InvalidArgument('Group id value error') area_id = body.get('area_id') areaname = body.get('areaname') show_id = body.get('show_id') if not areaname and not show_id: raise InvalidArgument('No value change') rpc = get_client() session = endpoint_session() query = model_query(session, GameArea, filter=GameArea.area_id == area_id) with session.begin(): area = query.one_or_none() if not area: raise InvalidArgument('No area found') if area.group_id != group_id: raise InvalidArgument('Area group not %d' % group_id) entityinfo = entity_controller.show(req=req, entity=area.entity, endpoint=common.NAME, body={'ports': False})['data'][0] agent_id = entityinfo['agent_id'] metadata = entityinfo['metadata'] if not metadata: raise InvalidArgument( 'Agent is off line, can not reset entity') if areaname: if model_count_with_key(session, GameArea, filter=and_( GameArea.group_id == group_id, GameArea.areaname == areaname)): raise InvalidArgument('Area name duplicate in group %d' % group_id) area.areaname = areaname if show_id: area.show_id = show_id target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME finishtime, timeout = rpcfinishtime() rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'change_entity_area', 'args': dict(entity=area.entity, area_id=area.area_id, show_id=area.show_id, areaname=area.areaname) }, timeout=timeout) if not rpc_ret: raise RpcResultError('change entity area result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('change entity area fail %s' % rpc_ret.get('result')) session.flush() return resultutils.results(result='change group areas success')
def rpc_asyncrequest(self, ctxt, asyncrequest, rpc_target, rpc_method, rpc_ctxt, rpc_args): """async respone check""" session = get_session() finishtime = ctxt.get('finishtime', None) asyncrequest = AsyncRequest(**asyncrequest) pre_run = ctxt.pop('pre_run', None) after_run = ctxt.pop('after_run', None) post_run = ctxt.pop('post_run', None) if finishtime and int(realnow()) >= finishtime: asyncrequest.resultcode = manager_common.RESULT_OVER_FINISHTIME asyncrequest.result = 'Async request over finish time' asyncrequest.status = manager_common.FINISH try: session.add(asyncrequest) session.flush() except DBDuplicateEntry: LOG.warning('Async request record DBDuplicateEntry') except DBError as e: LOG.error('Async request record DBError %s: %s' % (e.__class__.__name__, e.message)) return if not self.is_active: asyncrequest.resultcode = manager_common.SCHEDULER_STATUS_ERROR asyncrequest.result = 'Rpc server not active now' asyncrequest.status = manager_common.FINISH session.add(asyncrequest) session.flush() return try: if pre_run: pre_run = self._compile('pre', pre_run) if after_run: after_run = self._compile('after', after_run) if post_run: post_run = self._compile('post', post_run) except (KeyError, jsonutils.ValidationError): asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR asyncrequest.result = 'Rpc server can not find executer or ctxt error' asyncrequest.status = manager_common.FINISH session.add(asyncrequest) session.flush() return # except Exception: # LOG.exception('wtf') # raise if rpc_ctxt.get('agents') is None: wait_agents = [x[0] for x in model_query(session, Agent.agent_id, filter=Agent.status > manager_common.DELETED).all()] else: wait_agents = rpc_ctxt.get('agents') rpc_ctxt.update({'request_id': asyncrequest.request_id, 'expire': asyncrequest.expire, 'finishtime': asyncrequest.finishtime}) try: target = Target(**rpc_target) rpc = get_client() except Exception: LOG.error('Prepare rpc clinet error') asyncrequest.resultcode = manager_common.SCHEDULER_PREPARE_ERROR asyncrequest.result = 'Rpc server prepare rpc clinet error' asyncrequest.status = manager_common.FINISH session.add(asyncrequest) session.flush() return if pre_run: try: pre_run.run(asyncrequest, wait_agents) except RpcServerCtxtException as e: asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR asyncrequest.result = 'Rpc server ctxt pre function fail: %s' % e.message asyncrequest.status = manager_common.FINISH session.add(asyncrequest) session.flush() return session.add(asyncrequest) session.flush() LOG.debug('Try cast rpc method %s' % rpc_method) try: rpc.cast(target, ctxt=rpc_ctxt, msg={'method': rpc_method, 'args': rpc_args}) except AMQPDestinationNotFound: asyncrequest.resultcode = manager_common.SEND_FAIL asyncrequest.result = 'Async %s request send fail, AMQPDestinationNotFound' % rpc_method asyncrequest.status = manager_common.FINISH session.flush() return LOG.debug('Cast %s to %s success' % (asyncrequest.request_id, target.to_dict())) if after_run: try: after_run.run(asyncrequest, wait_agents) except RpcServerCtxtException as e: asyncrequest.result = 'Async request %s cast success, ' \ 'ctxt after function error~%s' % (rpc_method, e.message) else: asyncrequest.result = 'Async request %s cast success' % rpc_method finally: session.flush() request_id = asyncrequest.request_id finishtime = asyncrequest.finishtime deadline = asyncrequest.deadline + 1 expire = asyncrequest.expire if expire: storage = get_cache() else: storage = session def check_respone(): wait = finishtime - int(time.time()) # 先等待3秒,可以做一次提前检查 if wait > 3: eventlet.sleep(3) no_response_agents = set(wait_agents) interval = int(wait / 10) if interval < 3: interval = 3 elif interval > 10: interval = 10 not_overtime = 2 while True: no_response_agents = responeutils.norespones(storage=storage, request_id=request_id, agents=no_response_agents) if not no_response_agents: break if int(time.time()) < finishtime: eventlet.sleep(interval) if int(time.time()) > deadline: not_overtime -= 1 if not not_overtime: break eventlet.sleep(1) LOG.debug('Not response agents count %d' % len(no_response_agents)) bulk_data = [] agent_time = int(time.time()) for agent_id in no_response_agents: data = dict(request_id=request_id, agent_id=agent_id, agent_time=agent_time, server_time=agent_time, resultcode=manager_common.RESULT_OVER_FINISHTIME, result='Agent respone overtime') bulk_data.append(data) responeutils.bluk_insert(storage, no_response_agents, bulk_data, expire) asyncrequest.status = manager_common.FINISH if no_response_agents: asyncrequest.resultcode = manager_common.RESULT_NOT_ALL_SUCCESS asyncrequest.result = 'agents not respone, count:%d' % len(no_response_agents) else: asyncrequest.resultcode = manager_common.RESULT_SUCCESS asyncrequest.result = 'all agent respone result' session.flush() if post_run: try: post_run.run(asyncrequest, no_response_agents) except RpcServerCtxtException as e: asyncrequest.result += (' ctxt post function error~%s' % e.message) session.flush() session.close() threadpool.add_thread(safe_func_wrapper, check_respone, LOG)