def opentime(self, req, group_id, objtype, entity, body=None):
"""修改开服时间接口"""
body = body or {}
group_id = int(group_id)
entity = int(entity)
if objtype != common.GAMESERVER:
raise InvalidArgument('Api just for %s' % common.GAMESERVER)
opentime = int(body.pop('opentime'))
if opentime < 0 or opentime >= int(time.time()) + 86400 * 15:
raise InvalidArgument('opentime value error')
session = endpoint_session()
with session.begin():
query = model_query(session, AppEntity, filter=AppEntity.entity == entity)
_entity = query.one()
if _entity.objtype != objtype:
raise InvalidArgument('Entity is not %s' % objtype)
if _entity.group_id != group_id:
raise InvalidArgument('Entity group %d not match %d' % (_entity.group_id, group_id))
metadata, ports = self._entityinfo(req=req, entity=entity)
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.NAME
rpc = get_client()
finishtime, timeout = rpcfinishtime()
# with session.begin():
rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime},
msg={'method': 'opentime_entity',
'args': dict(entity=entity, opentime=opentime)},
timeout=timeout)
query.update({'opentime': opentime})
if not rpc_ret:
raise RpcResultError('change entity opentime result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('change entity opentime fail %s' % rpc_ret.get('result'))
return resultutils.results(result='change entity %d opentime success' % entity)
def _start_database(self, database, **kwargs):
req = kwargs.pop('req')
entity = int(database.reflection_id)
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
]
},
msg={
'method': 'start_entity',
'args': dict(entity=entity)
},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('create entitys result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('create entity fail %s' %
rpc_ret.get('result'))
return rpc_ret
def _revoke_database_user(self, database, auth, **kwargs):
req = kwargs.pop('req')
entity = int(database.reflection_id)
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
if not metadata:
raise InvalidArgument('Traget database agent is offline')
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
]
},
msg={
'method': 'revoke_entity',
'args': dict(entity=entity, auth=auth)
},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('revoke grant from database result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('revoke grant from database fail %s' %
rpc_ret.get('result'))
return rpc_ret
def wapper():
eventlet.sleep(random.randint(0, 5))
# save report log
session = get_session()
report = AgentReportLog(**snapshot)
session.add(report)
session.flush()
session.close()
process = snapshot.get('running') + snapshot.get('sleeping')
free = snapshot.get('free') + snapshot.get('cached')
conns = snapshot.get('syn') + snapshot.get('enable')
cputime = snapshot.get('iowait') + snapshot.get('user') \
+ snapshot.get('system') + snapshot.get('nice')\
+ snapshot.get('irq') + snapshot.get('sirq')
rpc = get_client()
# send to rpc server
rpc.cast(targetutils.target_rpcserver(fanout=True),
ctxt = {},
msg={'method': 'changesource',
'args': {'agent_id': agent_id,
'free': free,
'process': process,
'cputime': cputime,
'iowait': snapshot.get('iowait'),
'left': snapshot.get('left'),
'fds': snapshot.get('num_fds'),
'conns': conns,
'metadata': metadata,
}})
def wapper():
if self.delay:
eventlet.sleep(self.delay)
data = self.notify[keyword]
target = Target(**data.pop('target'))
func = getattr(get_client(), data.pop('method'))
func(target, **data)
def create(self, req, body=None):
body = body or {}
jsonutils.schema_validate(body, SCHEDULEJOBSCHEMA)
start = datetime.datetime.fromtimestamp(body['start'])
if start < int(time.time()) + 300:
raise InvalidArgument('Do not add a scheduler in 5 min')
job_id = uuidutils.Gkey()
rpc = get_client()
glock = get_global().lock('autorelase')
with glock(targetutils.schedule_job(), 30):
job_result = rpc.call(targetutils.target_anyone(
manager_common.SCHEDULER),
ctxt={'finishtime': ""},
msg={
'method': 'scheduler',
'args': {
'job_id': job_id,
'jobdata': body
}
}) # job interval
if not job_result:
raise RpcResultError('delete_agent_precommit result is None')
if job_result.get('resultcode') != manager_common.RESULT_SUCCESS:
return resultutils.results(result=job_result.get('result'))
return resultutils.results(
result='Create scheduler job:%d success' % job_id)
def notify_create(target, agent_id, entity, body, action='create'):
rpc = get_client()
body.setdefault('entity', entity)
finishtime = body.pop('finishtime', None)
if finishtime is None:
finishtime, timeout = rpcfinishtime()
else:
timeout = finishtime - int(time.time()) + 1
if timeout < 3:
raise InvalidArgument('Timeout less then 3')
create_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
],
'entitys': [
entity,
]
},
msg={
'method': '%s_entity' % action,
'args': body
},
timeout=timeout)
return create_ret
def delete(self, req, agent_id, body=None):
"""call buy agent"""
# if force is true
# will not notify agent, just delete agent from database
body = body or {}
force = body.get('force', False)
rpc = get_client()
global_data = get_global()
metadata = None
with global_data.delete_agent(agent_id) as agent:
if not force:
metadata = BaseContorller.agent_metadata(agent.agent_id)
if metadata is None:
raise RpcPrepareError('Can not delete offline agent, try force')
agent_ipaddr = metadata.get('local_ip')
secret = uuidutils.generate_uuid()
# tell agent wait delete
finishtime, timeout = rpcfinishtime()
delete_agent_precommit = rpc.call(targetutils.target_agent(agent),
ctxt={'finishtime': finishtime},
msg={'method': 'delete_agent_precommit',
'args': {'agent_id': agent.agent_id,
'agent_type': agent.agent_type,
'host': agent.host,
'agent_ipaddr': agent_ipaddr,
'secret': secret}
}, timeout=timeout)
if not delete_agent_precommit:
raise RpcResultError('delete_agent_precommit result is None')
if delete_agent_precommit.get('resultcode') != manager_common.RESULT_SUCCESS:
return resultutils.results(result=delete_agent_precommit.get('result'),
resultcode=manager_common.RESULT_ERROR)
# if not force:
# tell agent delete itself
finishtime = rpcfinishtime()[0]
LOG.info('Delete agent %s postcommit with secret %s' % (agent_ipaddr, secret))
rpc.cast(targetutils.target_agent(agent),
ctxt={'finishtime': finishtime},
msg={'method': 'delete_agent_postcommit',
'args': {'agent_id': agent.agent_id,
'agent_type': agent.agent_type,
'host': agent.host,
'agent_ipaddr': agent_ipaddr,
'secret': secret}})
def wapper():
rpc.cast(targetutils.target_rpcserver(fanout=True),
msg={'method': 'deletesource',
'args': {'agent_id': agent_id}})
threadpool.add_thread(safe_func_wrapper, wapper, LOG)
result = resultutils.results(result='Delete agent success',
data=[dict(agent_id=agent.agent_id,
host=agent.host,
status=agent.status,
metadata=metadata,
ports_range=jsonutils.safe_loads_as_bytes(agent.ports_range) or [])
])
return result
def _expire():
try:
BaseContorller._agent_metadata_expire(agent_id, expire + fix)
except AgentMetadataMiss:
# 元数据丢失, 通知agent重新上报
rpc = get_client()
# send to rpc server
rpc.cast(targetutils.target_all(fanout=True),
ctxt={'agents': [agent_id, ]},
msg={'method': 'flush_metadata',
'args': {'expire': expire}})
def _unbond_database(self, session, master, slave, relation, **kwargs):
req = kwargs.pop('req')
entity = int(slave.reflection_id)
with session.begin(subtransactions=True):
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
if not metadata:
raise InvalidArgument('Traget database agent is offline')
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
# 发送master信息到从库所在agent
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime + 3,
'agents': [
agent_id,
]
},
msg={
'method':
'unbond_entity',
'args':
dict(entity=entity,
force=kwargs.get('force', False),
master=dict(
database_id=master.database_id,
ready=relation.ready,
schemas=[
schema.schema
for schema in master.schemas
],
))
},
timeout=timeout + 3)
if not rpc_ret:
raise RpcResultError('unbond database result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('unbond database fail %s' %
rpc_ret.get('result'))
# 绑定状态设置就绪
session.delete(relation)
session.flush()
auth = privilegeutils.mysql_replprivileges(
slave.database_id, metadata.get('local_ip'))
auth['schema'] = '*'
return self._revoke_database_user(master, auth, req=req)
def _slave_database(self, session, master, slave, **kwargs):
req = kwargs.pop('req')
with session.begin(subtransactions=True):
# get slave host and port
_host, _port = self._get_entity(req=req,
entity=int(slave.reflection_id),
raise_error=True)
entity = int(master.reflection_id)
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
if not metadata:
raise InvalidArgument('Traget database agent is offline')
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
# 发送slave信息到主库所在agent
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime + 5,
'agents': [
agent_id,
]
},
msg={
'method':
'slave_entity',
'args':
dict(entity=entity,
schemas=kwargs.get('schemas'),
file=kwargs.get('file'),
position=kwargs.get('position'),
bond=dict(
database_id=slave.database_id,
host=_host,
port=_port))
},
timeout=timeout + 5)
if not rpc_ret:
raise RpcResultError(
'bond slave for master database result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('bond slave for master database fail %s' %
rpc_ret.get('result'))
return rpc_ret
def send_asyncrequest(asyncrequest,
rpc_target,
rpc_ctxt,
rpc_method,
rpc_args=None,
async_ctxt=None):
rpc = get_client()
session = get_session()
try:
rpc.cast(
targetutils.target_rpcserver(),
# ctxt={'finishtime': asyncrequest.finishtime-2},
ctxt=async_ctxt or {},
msg={
'method': 'asyncrequest',
'args': {
'asyncrequest': asyncrequest.to_dict(),
'rpc_target': rpc_target.to_dict(),
'rpc_method': rpc_method,
'rpc_ctxt': rpc_ctxt,
'rpc_args': rpc_args or dict()
}
})
except AMQPDestinationNotFound as e:
LOG.error('Send async request to scheduler fail %s' %
e.__class__.__name__)
asyncrequest.status = manager_common.FINISH
asyncrequest.result = e.message
asyncrequest.resultcode = manager_common.SCHEDULER_NOTIFY_ERROR
try:
session.add(asyncrequest)
session.flush()
except DBDuplicateEntry:
LOG.warning(
'Async request rpc call result is None, but recode found')
except Exception as e:
if LOG.isEnabledFor(logging.DEBUG):
LOG.exception('Async request rpc cast fail')
else:
LOG.error('Async request rpc cast unkonw error')
asyncrequest.status = manager_common.FINISH
asyncrequest.result = 'Async request rpc cast error: %s' % e.__class__.__name__
asyncrequest.resultcode = manager_common.RESULT_ERROR
try:
session.add(asyncrequest)
session.flush()
raise
except DBDuplicateEntry:
LOG.warning(
'Async request rpc call result is None, but recode found')
def _ready_relation(self, session, master, slave, relation, **kwargs):
req = kwargs.pop('req')
entity = int(slave.reflection_id)
with session.begin(subtransactions=True):
schemas = [schema.schema for schema in master.schemas]
_host, _port = self._get_entity(req,
int(master.reflection_id),
raise_error=True)
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
if not metadata:
raise InvalidArgument('Traget database agent is offline')
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
# 发送master信息到从库所在agent
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
]
},
msg={
'method':
'entity_replication_ready',
'args':
dict(entity=entity,
master=dict(
database_id=master.database_id,
host=_host,
port=_port,
schemas=schemas))
},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('get replication status result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('get replication status fail %s' %
rpc_ret.get('result'))
# 绑定状态设置就绪
relation.ready = True
return rpc_ret
def _bond_database(self, session, master, slave, relation, **kwargs):
req = kwargs.pop('req')
entity = int(slave.reflection_id)
with session.begin(subtransactions=True):
_entity = entity_controller.show(req=req,
entity=entity,
endpoint=common.DB,
body={'ports': False})['data'][0]
agent_id = _entity['agent_id']
metadata = _entity['metadata']
if not metadata:
raise InvalidArgument('Traget database agent is offline')
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.DB
rpc = get_client()
finishtime, timeout = rpcfinishtime()
# 发送master信息到从库所在agent
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime + 5,
'agents': [
agent_id,
]
},
msg={
'method':
'bond_entity',
'args':
dict(entity=entity,
force=kwargs.get('force', False),
master=dict(
database_id=master.database_id,
host=kwargs.get('host'),
port=kwargs.get('port'),
passwd=kwargs.get('passwd'),
file=kwargs.get('file'),
position=kwargs.get('position'),
schemas=kwargs.get('schemas')))
},
timeout=timeout + 5)
if not rpc_ret:
raise RpcResultError('bond database result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('bond database fail %s' %
rpc_ret.get('result'))
# 绑定状态设置就绪
relation.ready = True if not kwargs.get('schemas') else False
return rpc_ret
def _check_file(agent_id, objtype, appfile):
metadata = BaseContorller.agent_metadata(agent_id)
if not metadata:
return False
target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host'))
rpc = get_client()
rpc_ret = rpc.call(target, ctxt={'agents': [agent_id, ]},
msg={'method': 'check_file',
'args': dict(objtype=objtype, appfile=appfile)})
if not rpc_ret:
LOG.error('Rpc call result is None')
return False
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
return False
return True
def continues(self, req, uuid, body=None):
"""中途失败的合服任务再次运行"""
session = endpoint_session()
query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid)
query = query.options(joinedload(MergeTask.entitys, innerjoin=False))
etask = query.one()
if etask.status == common.MERGEFINISH:
raise InvalidArgument('Merge task has all ready finished')
_query = model_query(session,
AppEntity,
filter=AppEntity.entity == etask.entity)
_query = _query.options(
joinedload(AppEntity.databases, innerjoin=False))
appentity = _query.one_or_none()
if not appentity or not appentity.databases or appentity.objtype != common.GAMESERVER:
LOG.error('Etask entity can not be found or type/database error')
raise exceptions.MergeException(
'Etask entity can not be found or type/database error')
databases = self._database_to_dict(appentity)
rpc = get_client()
metadata, ports = self._entityinfo(req=req, entity=appentity.entity)
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.NAME
rpc_ret = rpc.call(target,
ctxt={'agents': [
appentity.agent_id,
]},
msg={
'method':
'continue_merge',
'args':
dict(entity=etask.entity,
uuid=uuid,
databases=databases)
})
if not rpc_ret:
raise RpcResultError('continue entity result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('continue entity fail %s' %
rpc_ret.get('result'))
return resultutils.results(
result='continue merge task command has been send',
data=[dict(uuid=etask.uuid, entity=etask.entity)])
def chioces(endpoint, includes=None, weighters=None):
"""return a agents list sort by weigher"""
rpc = get_client()
chioces_result = rpc.call(targetutils.target_rpcserver(),
ctxt=dict(),
msg={
'method': 'chioces',
'args': {
'target': endpoint,
'includes': includes,
'weighters': weighters
}
})
if not chioces_result:
raise RpcResultError('Active agent chioces result is None')
if chioces_result.pop('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('Call agent chioces fail: ' +
chioces_result.get('result'))
return chioces_result['agents']
def notify_delete(target, agent_id, entity, body, action='delete'):
rpc = get_client()
body.setdefault('entity', entity)
token = body.get('token')
finishtime = body.pop('finishtime', None)
if finishtime is None:
finishtime, timeout = rpcfinishtime()
else:
timeout = finishtime - int(time.time()) + 1
if timeout < 3:
raise InvalidArgument('Timeout less then 3')
if token:
# send a delete token
rpc.cast(target,
ctxt={
'finishtime': finishtime,
'entitys': [
entity,
]
},
msg={
'method': 'entity_token',
'args': {
'entity': entity,
'token': token
}
})
delete_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
],
'entitys': [
entity,
]
},
msg={
'method': '%s_entity' % action,
'args': body
},
timeout=timeout)
return delete_ret
def readlog(self, req, agent_id, body=None):
"""call by agent"""
body = body or {}
lines = body.get('lines', 10)
rpc = get_client()
metadata = BaseContorller.agent_metadata(agent_id)
if metadata is None:
raise RpcPrepareError('Can not get log from offline agent: %d' % agent_id)
session = get_session()
query = model_query(session, Agent, filter=Agent.agent_id == agent_id)
agent = query.one()
rpc_ret = rpc.call(targetutils.target_agent(agent),
ctxt={'finishtime': rpcfinishtime()[0]},
msg={'method': 'readlog', 'args': {'lines': lines}})
if not rpc_ret:
raise RpcResultError('Get log agent rpc result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('Get log agent rpc result: ' + rpc_ret.get('result'))
return resultutils.results(result=rpc_ret.get('result'), data=[rpc_ret.get('uri')])
def readlog(self, req, endpoint, entity, body=None):
body = body or {}
path = body.get('path')
lines = body.get('lines', 10)
if not path or '..' in path:
raise InvalidArgument('Path value error')
endpoint = validateutils.validate_endpoint(endpoint)
entity = int(entity)
session = get_session(readonly=True)
query = model_query(session,
AgentEntity,
filter=and_(AgentEntity.endpoint == endpoint,
AgentEntity.entity == entity))
_entity = query.one_or_none()
if not _entity:
raise InvalidArgument('no entity found for %s' % endpoint)
metadata = BaseContorller.agent_metadata(_entity.agent_id)
if not metadata:
raise InvalidArgument('Can not get log from off line agent')
target = targetutils.target_agent_by_string(manager_common.APPLICATION,
metadata.get('host'))
target.namespace = endpoint
rpc = get_client()
rpc_ret = rpc.call(target,
ctxt={'finishtime': rpcfinishtime()[0]},
msg={
'method': 'readlog',
'args': {
'entity': entity,
'path': path,
'lines': lines
}
})
if not rpc_ret:
raise RpcResultError('Get %s.%d log rpc result is None' %
(endpoint, entity))
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise InvalidArgument(('Get %s.%d log agent rpc result: ' %
(endpoint, entity)) + rpc_ret.get('result'))
return resultutils.results(result=rpc_ret.get('result'),
data=[rpc_ret.get('uri')])
def active(self, req, agent_id, body=None):
"""call buy client"""
body = body or {}
status = body.get('status', manager_common.ACTIVE)
if status not in (manager_common.ACTIVE, manager_common.UNACTIVE):
raise InvalidArgument('Argument status not right')
rpc = get_client()
session = get_session()
query = model_query(session, Agent,
filter=and_(Agent.agent_id == agent_id,
Agent.status > manager_common.DELETED))
agent = query.one()
# make sure agent is online
metadata = BaseContorller.agent_metadata(agent.agent_id)
if metadata is None:
raise RpcPrepareError('Can not active or unactive a offline agent: %d' % agent_id)
agent_ipaddr = metadata.get('local_ip')
with session.begin():
agent.update({'status': status})
active_agent = rpc.call(targetutils.target_agent(agent),
ctxt={'finishtime': rpcfinishtime()[0]},
msg={'method': 'active_agent',
'args': {'agent_id': agent_id,
'agent_ipaddr': agent_ipaddr,
'status': status}
})
if not active_agent:
raise RpcResultError('Active agent rpc result is None')
if active_agent.pop('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('Call agent active or unactive fail: ' + active_agent.get('result'))
result = resultutils.results(result=active_agent.pop('result'),
data=[dict(agent_id=agent.agent_id,
host=agent.host,
agent_type=agent.agent_type,
metadata=metadata,
status=agent.status)
])
return result
def post_create_entity(self, entity, endpoint, **kwargs):
entity = int(entity)
endpoint = validateutils.validate_endpoint(endpoint)
session = get_session(readonly=True)
query = model_query(session,
AgentEntity,
filter=and_(AgentEntity.endpoint == endpoint,
AgentEntity.entity == entity))
_entity = query.one()
metadata = BaseContorller.agent_metadata(_entity.agent_id)
if not metadata:
raise RpcPrepareError('Agent not online, can not sen post create')
target = targetutils.target_agent_by_string(
metadata.get('agent_type'),
metadata.get('host'),
)
target.namespace = endpoint
body = dict(entity=entity)
body.update(kwargs)
rpc = get_client()
rpc.cast(target,
ctxt={
'finishtime': body.pop('finishtime',
rpcfinishtime()[0]),
'entitys': [
entity,
]
},
msg={
'method': 'post_create_entity',
'args': body
})
return resultutils.results(result='notify post create success',
data=[
dict(entity=entity,
agent_id=_entity.agent_id,
endpoint=endpoint)
])
def swallow(self, req, entity, body=None):
"""合服内部接口,一般由agent调用
用于新实体吞噬旧实体的区服和数据库"""
body = body or {}
entity = int(entity)
uuid = body.get('uuid')
if not uuid:
raise InvalidArgument('Merger uuid is None')
session = endpoint_session()
query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid)
query = query.options(joinedload(MergeTask.entitys, innerjoin=False))
glock = get_gamelock()
rpc = get_client()
with session.begin():
etask = query.one_or_none()
if not etask:
raise InvalidArgument('Not task exit with %s' % uuid)
# 新实体不匹配
if etask.entity != body.get('entity'):
raise InvalidArgument('New entity not %d' % etask.entity)
# 找到目标实体
appentity = None
for _entity in etask.entitys:
if _entity.entity == entity:
if _entity.status != common.MERGEING:
if _entity.status != common.SWALLOWING:
raise InvalidArgument(
'Swallow entity find status error')
if not _entity.databases or not _entity.areas:
raise InvalidArgument(
'Entity is swallowing but database or ares is None'
)
LOG.warning('Entit is swallowing, return saved data')
return resultutils.results(
result='swallow entity is success',
data=[
dict(databases=jsonutils.loads_as_bytes(
_entity.databases),
areas=jsonutils.loads_as_bytes(
_entity.areas))
])
_query = model_query(session,
AppEntity,
filter=AppEntity.entity == entity)
_query = _query.options(
joinedload(AppEntity.databases, innerjoin=False))
appentity = _query.one_or_none()
break
if not appentity:
raise InvalidArgument('Can not find app entity?')
if appentity.objtype != common.GAMESERVER:
raise InvalidArgument('objtype error, entity not %s' %
common.GAMESERVER)
if appentity.status != common.MERGEING:
raise InvalidArgument('find status error, when swallowing')
databases = self._database_to_dict(appentity)
areas = [area.to_dict() for area in appentity.areas]
if not databases or not areas:
LOG.error('Entity no areas or databases record')
return resultutils.results(
result='swallow entity fail, '
'target entity can not found database or areas',
resultcode=manager_common.RESULT_ERROR)
with glock.grouplock(group=appentity.group_id):
# 发送吞噬命令到目标区服agent
metadata, ports = self._entityinfo(req=req, entity=entity)
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.NAME
rpc_ret = rpc.call(target,
ctxt={'agents': [
appentity.agent_id,
]},
msg={
'method': 'swallow_entity',
'args': dict(entity=entity)
})
if not rpc_ret:
raise RpcResultError('swallow entity result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('swallow entity fail %s' %
rpc_ret.get('result'))
# 修改实体在合服任务中的状态,存储areas以及databases
appentity.status = common.SWALLOWING
_entity.status = common.SWALLOWING
_entity.areas = jsonutils.dumps(areas)
_entity.databases = jsonutils.dumps(databases)
session.flush()
return resultutils.results(
result='swallow entity is success',
data=[dict(databases=databases, areas=areas)])
def swallowed(self, req, entity, body=None):
"""
合服内部接口,一般由agent调用
用于新实体吞噬旧实体的区服完成后调用
调用后将设置appentity为deleted状态
"""
body = body or {}
entity = int(entity)
uuid = body.get('uuid')
if not uuid:
raise InvalidArgument('Merger uuid is None')
session = endpoint_session()
query = model_query(session, MergeTask, filter=MergeTask.uuid == uuid)
query = query.options(joinedload(MergeTask.entitys, innerjoin=False))
glock = get_gamelock()
rpc = get_client()
appentity = None
with session.begin():
etask = query.one_or_none()
if not etask:
raise InvalidArgument('Not task exit with %s' % uuid)
# 新实体不匹配
if etask.entity != body.get('entity'):
raise InvalidArgument('New entity not %d' % etask.entity)
for _entity in etask.entitys:
if _entity.entity == entity:
if _entity.status != common.SWALLOWING:
raise InvalidArgument(
'Swallowed entity find status error')
_query = model_query(session,
AppEntity,
filter=AppEntity.entity == entity)
_query = _query.options(
joinedload(AppEntity.databases, innerjoin=False))
appentity = _query.one_or_none()
break
if not appentity:
raise InvalidArgument('Can not find app entity?')
if appentity.objtype != common.GAMESERVER:
raise InvalidArgument('objtype error, entity not %s' %
common.GAMESERVER)
if appentity.status != common.SWALLOWING:
raise InvalidArgument('find status error, when swallowed')
with glock.grouplock(group=appentity.group_id):
# 发送吞噬完成命令到目标区服agent
metadata, ports = self._entityinfo(req=req, entity=entity)
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.NAME
rpc_ret = rpc.call(target,
ctxt={'agents': [
appentity.agent_id,
]},
msg={
'method': 'swallowed_entity',
'args': dict(entity=entity)
})
if not rpc_ret:
raise RpcResultError('swallowed entity result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('swallowed entity fail %s' %
rpc_ret.get('result'))
# appentity状态修改为deleted
appentity.status = common.DELETED
# 修改实体在合服任务中的状态
_entity.status = common.MERGEED
session.flush()
# area绑定新实体
_query = model_query(session,
GameArea,
filter=GameArea.entity == entity)
_query.update({'entity': etask.entity})
session.flush()
def _unquote():
LOG.info('Swallowed %d finish, try unquote database' %
appentity.entity)
for database in appentity.databases:
try:
schema_controller.unquote(req, quote_id=database.quote_id)
except Exception:
LOG.error('Delete database quote fail')
eventlet.spawn_n(_unquote)
return resultutils.results(
result='swallowed entity is success',
data=[
dict(databases=jsonutils.loads_as_bytes(_entity.databases),
areas=jsonutils.loads_as_bytes(_entity.areas))
])
def clean(self, req, group_id, objtype, entity, body=None):
"""彻底删除entity"""
body = body or {}
action = body.pop('clean', 'unquote')
force = False
ignores = body.pop('ignores', [])
if action not in ('delete', 'unquote', 'force'):
raise InvalidArgument('clean option value error')
if action == 'force':
action = 'delete'
force = True
group_id = int(group_id)
entity = int(entity)
session = endpoint_session()
glock = get_gamelock()
metadata, ports = self._entityinfo(req=req, entity=entity)
if not metadata:
raise InvalidArgument('Agent offline, can not delete entity')
query = model_query(session, AppEntity, filter=AppEntity.entity == entity)
query = query.options(joinedload(AppEntity.databases, innerjoin=False))
_entity = query.one()
rollbacks = []
def _rollback():
for back in rollbacks:
__database_id = back.get('database_id')
__schema = back.get('schema')
__quote_id = back.get('quote_id')
rbody = dict(quote_id=__quote_id, entity=entity)
rbody.setdefault(dbcommon.ENDPOINTKEY, common.NAME)
try:
schema_controller.bond(req, database_id=__database_id, schema=__schema, body=rbody)
except Exception:
LOG.error('rollback entity %d quote %d.%s.%d fail' %
(entity, __database_id, schema, __quote_id))
with glock.grouplock(group=group_id):
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.NAME
rpc = get_client()
finishtime, timeout = rpcfinishtime()
LOG.warning('Clean entity %s.%d with action %s' % (objtype, entity, action))
with session.begin():
rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime},
msg={'method': 'stoped', 'args': dict(entity=entity)})
if not rpc_ret:
raise RpcResultError('check entity is stoped result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('check entity is stoped fail, running')
with session.begin():
if _entity.status != common.DELETED:
raise InvalidArgument('Entity status is not DELETED, '
'mark status to DELETED before delete it')
if _entity.objtype != objtype:
raise InvalidArgument('Objtype not match')
if _entity.group_id != group_id:
raise InvalidArgument('Group id not match')
# esure database delete
if action == 'delete':
LOG.warning('Clean option is delete, can not rollback when fail')
if not force:
for _database in _entity.databases:
schema = '%s_%s_%s_%d' % (common.NAME, objtype, _database.subtype, entity)
schema_info = schema_controller.show(req=req, database_id=_database.database_id,
schema=schema,
body={'quotes': True})['data'][0]
quotes = {}
for _quote in schema_info['quotes']:
quotes[_quote.get('quote_id')] = _quote.get('desc')
if _database.quote_id not in quotes.keys():
# if set(quotes) != set([_database.quote_id]):
result = 'delete %s:%d fail' % (objtype, entity)
reason = ': database [%d].%s quote: %s' % (_database.database_id, schema, str(quotes))
return resultutils.results(result=(result + reason),
resultcode=manager_common.RESULT_ERROR)
quotes.pop(_database.quote_id)
for quote_id in quotes.keys():
if quotes[quote_id] in ignores:
quotes.pop(quote_id, None)
if quotes:
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug('quotes not match for %d: %s' % (schema_info['schema_id'],
schema))
for quote_id in quotes.keys():
LOG.debug('quote %d: %s exist' % (quote_id, quotes[quote_id]))
LOG.debug('Can not delete schema before delete quotes')
return resultutils.results(result='Quotes not match',
resultcode=manager_common.RESULT_ERROR)
LOG.info('Databae quotes check success for %s' % schema)
# clean database
for _database in _entity.databases:
schema = '%s_%s_%s_%d' % (common.NAME, objtype, _database.subtype, entity)
if action == 'delete':
LOG.warning('Delete schema %s from %d' % (schema, _database.database_id))
try:
schema_controller.delete(req=req, database_id=_database.database_id,
schema=schema, body={'unquotes': [_database.quote_id],
'ignores': ignores, 'force': force})
except GopdbError as e:
LOG.error('Delete schema:%s from %d fail, %s' % (schema, _database.database_id,
e.message))
if not force:
raise e
except Exception:
LOG.exception('Delete schema:%s from %d fail' % (schema, _database.database_id))
if not force:
raise
elif action == 'unquote':
LOG.info('Try unquote %d' % _database.quote_id)
try:
quote = schema_controller.unquote(req=req, quote_id=_database.quote_id)['data'][0]
if quote.get('database_id') != _database.database_id:
LOG.critical('quote %d with database %d, not %d' % (_database.quote_id,
quote.get('database_id'),
_database.database_id))
raise RuntimeError('Data error, quote database not the same')
rollbacks.append(dict(database_id=_database.database_id,
quote_id=_database.quote_id, schema=schema))
except Exception as e:
LOG.error('Unquote %d fail, try rollback' % _database.quote_id)
if not force:
threadpool.add_thread(_rollback)
raise e
token = uuidutils.generate_uuid()
LOG.info('Send delete command with token %s' % token)
session.delete(_entity)
session.flush()
try:
entity_controller.delete(req, common.NAME, entity=entity, body=dict(token=token))
except Exception as e:
# roll back unquote
threadpool.add_thread(_rollback)
raise e
return resultutils.results(result='delete %s:%d success' % (objtype, entity),
data=[dict(entity=entity, objtype=objtype,
ports=ports, metadata=metadata)])
def delete(self, req, group_id, objtype, entity, body=None):
"""标记删除entity"""
body = body or {}
force = body.get('force', False)
group_id = int(group_id)
entity = int(entity)
session = endpoint_session()
glock = get_gamelock()
metadata, ports = self._entityinfo(req=req, entity=entity)
if not metadata:
raise InvalidArgument('Agent offline, can not delete entity')
query = model_query(session,
AppEntity,
filter=AppEntity.entity == entity)
if objtype == common.GAMESERVER:
query = query.options(joinedload(AppEntity.areas, innerjoin=False))
_entity = query.one()
if _entity.status == common.DELETED:
return resultutils.results(result='mark %s entity delete success' %
objtype,
data=[
dict(entity=entity,
objtype=objtype,
ports=ports,
metadata=metadata)
])
if _entity.objtype != objtype:
raise InvalidArgument('Objtype not match')
if _entity.group_id != group_id:
raise InvalidArgument('Group id not match')
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.NAME
rpc = get_client()
with glock.grouplock(group=group_id):
if objtype == common.GMSERVER:
if model_count_with_key(
session, AppEntity, filter=AppEntity.group_id
== group_id) > 1:
raise InvalidArgument(
'You must delete other objtype entity before delete gm'
)
if model_count_with_key(
session, Package, filter=Package.group_id
== group_id) > 1:
raise InvalidArgument(
'You must delete other Package before delete gm')
elif objtype == common.CROSSSERVER:
if model_count_with_key(
session,
AppEntity,
filter=AppEntity.cross_id == _entity.entity):
raise InvalidArgument('Cross server are reflected')
with session.begin():
# 确认实体没有运行
rpc_ret = rpc.call(target,
ctxt={'agents': [
_entity.agent_id,
]},
msg={
'method': 'stoped',
'args': dict(entity=entity)
})
if not rpc_ret:
raise RpcResultError('check entity stoped result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('check entity fail %s' %
rpc_ret.get('result'))
_entity.status = common.DELETED
session.flush()
if objtype == common.GAMESERVER:
# 删除所有资源版本引用
if _entity.versions:
for quote in six.itervalues(
jsonutils.loads_as_bytes(_entity.versions)):
threadpool.add_thread(cdnquote_controller.delete,
req, quote.get('quote_id'))
_entity.versions = None
session.flush()
if _entity.areas:
if len(_entity.areas) > 1:
raise InvalidArgument('%s areas more then one' %
objtype)
area = _entity.areas[0]
if not force:
if _entity.entity != model_max_with_key(
session,
AppEntity.entity,
filter=and_(
AppEntity.objtype == common.GAMESERVER,
AppEntity.group_id == group_id)):
raise InvalidArgument(
'entity %d is not the last gamesvr entity in group'
% entity)
session.flush()
session.delete(area)
session.flush()
_query = model_query(
session,
PackageArea,
filter=PackageArea.area_id == area.area_id)
_query.delete()
session.flush()
rpc.cast(target,
ctxt={'agents': [
_entity.agent_id,
]},
msg={
'method': 'change_status',
'args': dict(entity=entity, status=common.DELETED)
})
return resultutils.results(result='mark %s entity delete success' %
objtype,
data=[
dict(entity=entity,
objtype=objtype,
ports=ports,
metadata=metadata)
])
def reset(self, req, group_id, objtype, entity, body=None):
"""重置Entity程序以及配置"""
body = body or {}
group_id = int(group_id)
entity = int(entity)
# 重置程序文件,为空表示不需要重置程序文件
appfile = body.pop(common.APPFILE, None)
# 重置数据库信息
databases = body.pop('databases', False)
# 重置主服务器信息(gameserver专用)
chiefs = body.pop('chiefs', False)
# 查询entity信息
session = endpoint_session()
query = model_query(session, AppEntity, filter=AppEntity.entity == entity)
query = query.options(joinedload(AppEntity.databases, innerjoin=False))
_entity = query.one()
if _entity.objtype != objtype:
raise InvalidArgument('Entity is not %s' % objtype)
if _entity.group_id != group_id:
raise InvalidArgument('Entity group %d not match %d' % (_entity.group_id, group_id))
entityinfo = entity_controller.show(req=req, entity=entity,
endpoint=common.NAME,
body={'ports': False})['data'][0]
agent_id = entityinfo['agent_id']
metadata = entityinfo['metadata']
if not metadata:
raise InvalidArgument('Agent is off line, can not reset entity')
# 需要更新数据库
if databases:
miss = []
databases = {}
# 从本地查询数据库信息
for database in _entity.databases:
subtype = database.subtype
schema = '%s_%s_%s_%d' % (common.NAME, objtype, subtype, entity)
databases[subtype] = dict(host=database.host,
port=database.port,
user=database.user,
passwd=database.passwd,
schema=schema,
character_set=database.character_set)
# 必要数据库信息
NEEDED = common.DBAFFINITYS[objtype].keys()
# 数据库信息不匹配,从gopdb接口反查数据库信息
if set(NEEDED) != set(databases.keys()):
LOG.warning('Database not match, try find schema info from gopdb')
quotes = schema_controller.quotes(req, body=dict(entitys=[entity, ],
endpoint=common.NAME))['data']
for subtype in NEEDED:
if subtype not in databases:
# 从gopdb接口查询引用信息
schema = '%s_%s_%s_%d' % (common.NAME, objtype, subtype, entity)
for quote_detail in quotes:
# 确认引用不是从库且结构名称相等
if quote_detail['qdatabase_id'] == quote_detail['database_id'] \
and quote_detail['schema'] == schema:
databases.setdefault(subtype,
dict(host=quote_detail['host'],
port=quote_detail['port'],
user=quote_detail['user'],
passwd=quote_detail['passwd'],
schema=schema,
character_set=quote_detail['character_set']))
miss.append(AreaDatabase(quote_id=quote_detail['quote_id'],
database_id=quote_detail['qdatabase_id'],
entity=entity,
subtype=subtype,
host=quote_detail['host'], port=quote_detail['port'],
user=quote_detail['user'], passwd=quote_detail['passwd'],
ro_user=quote_detail['ro_user'],
ro_passwd=quote_detail['ro_passwd'],
character_set=quote_detail['character_set'])
)
quotes.remove(quote_detail)
break
if subtype not in databases:
LOG.critical('Miss database of %s' % schema)
# 数据库信息无法从gopdb中反查到
raise ValueError('Not %s.%s database found for %d' % (objtype, subtype, entity))
self._validate_databases(objtype, databases)
# 有数据库信息遗漏
if miss:
with session.begin():
for obj in miss:
session.add(obj)
session.flush()
if objtype == common.GAMESERVER and chiefs:
chiefs = {}
cross_id = _entity.cross_id
if cross_id is None:
raise ValueError('%s.%d cross_id is None' % (objtype, entity))
query = model_query(session, AppEntity,
filter=and_(AppEntity.group_id == group_id,
or_(AppEntity.entity == cross_id,
AppEntity.objtype == common.GMSERVER)))
_chiefs = query.all()
if len(_chiefs) != 2:
raise ValueError('Try find %s.%d chiefs from local database error' % (objtype, entity))
for chief in _chiefs:
for _objtype in (common.GMSERVER, common.CROSSSERVER):
_metadata, ports = self._entityinfo(req, chief.entity)
if not _metadata:
raise InvalidArgument('Metadata of %s.%d is none' % (_objtype, chief.entity))
if chief.objtype == _objtype:
chiefs[_objtype] = dict(entity=chief.entity,
ports=ports,
local_ip=_metadata.get('local_ip'))
if len(chiefs) != 2:
raise ValueError('%s.%d chiefs error' % (objtype, entity))
target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.NAME
rpc = get_client()
finishtime, timeout = rpcfinishtime()
if appfile:
finishtime += 30
timeout += 35
rpc_ret = rpc.call(target, ctxt={'finishtime': finishtime, 'agents': [agent_id, ]},
msg={'method': 'reset_entity',
'args': dict(entity=entity, appfile=appfile,
opentime=_entity.opentime,
databases=databases, chiefs=chiefs)},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('reset entity result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('reset entity fail %s' % rpc_ret.get('result'))
return resultutils.results(result='reset entity %d success' % entity)
def update(self, req, group_id, objtype, entity, body=None):
body = body or {}
group_id = int(group_id)
entity = int(entity)
status = body.get('status', common.OK)
if status not in (common.UNACTIVE, common.OK):
raise InvalidArgument('Status not in 0, 1, 2')
session = endpoint_session()
glock = get_gamelock()
query = model_query(session,
AppEntity,
filter=AppEntity.entity == entity)
if objtype == common.GAMESERVER:
query = query.options(joinedload(AppEntity.areas, innerjoin=False))
_entity = query.one()
if status == _entity.status:
return resultutils.results(result='%s entity status in same' %
objtype)
if _entity.status not in (common.OK, common.UNACTIVE):
return resultutils.results(
resultcode=manager_common.RESULT_ERROR,
result='%s entity is not ok or unactive' % objtype)
if _entity.objtype != objtype:
raise InvalidArgument('Objtype not match')
if _entity.group_id != group_id:
raise InvalidArgument('Group id not match')
entityinfo = entity_controller.show(req=req,
entity=entity,
endpoint=common.NAME,
body={'ports': False})['data'][0]
agent_id = entityinfo['agent_id']
metadata = entityinfo['metadata']
if not metadata:
raise InvalidArgument('Agent is off line, can not reset entity')
rpc = get_client()
target = targetutils.target_agent_by_string(metadata.get('agent_type'),
metadata.get('host'))
target.namespace = common.NAME
if objtype == common.GAMESERVER:
lock = functools.partial(
glock.arealock,
group=group_id,
areas=[area.area_id for area in _entity.areas])
else:
lock = functools.partial(glock.grouplock, group=group_id)
with lock():
with session.begin():
_entity.status = status
session.flush()
finishtime, timeout = rpcfinishtime()
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
]
},
msg={
'method': 'change_status',
'args': dict(entity=entity,
status=status)
},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('change entity sttus result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('change entity status fail %s' %
rpc_ret.get('result'))
return resultutils.results(result='%s entity update success' % objtype)
def area(self, req, group_id, body=None):
"""change entity area"""
body = body or {}
try:
group_id = int(group_id)
except (TypeError, ValueError):
raise InvalidArgument('Group id value error')
area_id = body.get('area_id')
areaname = body.get('areaname')
show_id = body.get('show_id')
if not areaname and not show_id:
raise InvalidArgument('No value change')
rpc = get_client()
session = endpoint_session()
query = model_query(session,
GameArea,
filter=GameArea.area_id == area_id)
with session.begin():
area = query.one_or_none()
if not area:
raise InvalidArgument('No area found')
if area.group_id != group_id:
raise InvalidArgument('Area group not %d' % group_id)
entityinfo = entity_controller.show(req=req,
entity=area.entity,
endpoint=common.NAME,
body={'ports':
False})['data'][0]
agent_id = entityinfo['agent_id']
metadata = entityinfo['metadata']
if not metadata:
raise InvalidArgument(
'Agent is off line, can not reset entity')
if areaname:
if model_count_with_key(session,
GameArea,
filter=and_(
GameArea.group_id == group_id,
GameArea.areaname == areaname)):
raise InvalidArgument('Area name duplicate in group %d' %
group_id)
area.areaname = areaname
if show_id:
area.show_id = show_id
target = targetutils.target_agent_by_string(
metadata.get('agent_type'), metadata.get('host'))
target.namespace = common.NAME
finishtime, timeout = rpcfinishtime()
rpc_ret = rpc.call(target,
ctxt={
'finishtime': finishtime,
'agents': [
agent_id,
]
},
msg={
'method':
'change_entity_area',
'args':
dict(entity=area.entity,
area_id=area.area_id,
show_id=area.show_id,
areaname=area.areaname)
},
timeout=timeout)
if not rpc_ret:
raise RpcResultError('change entity area result is None')
if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS:
raise RpcResultError('change entity area fail %s' %
rpc_ret.get('result'))
session.flush()
return resultutils.results(result='change group areas success')
def rpc_asyncrequest(self, ctxt,
asyncrequest, rpc_target, rpc_method,
rpc_ctxt, rpc_args):
"""async respone check"""
session = get_session()
finishtime = ctxt.get('finishtime', None)
asyncrequest = AsyncRequest(**asyncrequest)
pre_run = ctxt.pop('pre_run', None)
after_run = ctxt.pop('after_run', None)
post_run = ctxt.pop('post_run', None)
if finishtime and int(realnow()) >= finishtime:
asyncrequest.resultcode = manager_common.RESULT_OVER_FINISHTIME
asyncrequest.result = 'Async request over finish time'
asyncrequest.status = manager_common.FINISH
try:
session.add(asyncrequest)
session.flush()
except DBDuplicateEntry:
LOG.warning('Async request record DBDuplicateEntry')
except DBError as e:
LOG.error('Async request record DBError %s: %s' % (e.__class__.__name__, e.message))
return
if not self.is_active:
asyncrequest.resultcode = manager_common.SCHEDULER_STATUS_ERROR
asyncrequest.result = 'Rpc server not active now'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
try:
if pre_run:
pre_run = self._compile('pre', pre_run)
if after_run:
after_run = self._compile('after', after_run)
if post_run:
post_run = self._compile('post', post_run)
except (KeyError, jsonutils.ValidationError):
asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR
asyncrequest.result = 'Rpc server can not find executer or ctxt error'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
# except Exception:
# LOG.exception('wtf')
# raise
if rpc_ctxt.get('agents') is None:
wait_agents = [x[0] for x in model_query(session, Agent.agent_id,
filter=Agent.status > manager_common.DELETED).all()]
else:
wait_agents = rpc_ctxt.get('agents')
rpc_ctxt.update({'request_id': asyncrequest.request_id,
'expire': asyncrequest.expire,
'finishtime': asyncrequest.finishtime})
try:
target = Target(**rpc_target)
rpc = get_client()
except Exception:
LOG.error('Prepare rpc clinet error')
asyncrequest.resultcode = manager_common.SCHEDULER_PREPARE_ERROR
asyncrequest.result = 'Rpc server prepare rpc clinet error'
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
if pre_run:
try:
pre_run.run(asyncrequest, wait_agents)
except RpcServerCtxtException as e:
asyncrequest.resultcode = manager_common.SCHEDULER_EXECUTER_ERROR
asyncrequest.result = 'Rpc server ctxt pre function fail: %s' % e.message
asyncrequest.status = manager_common.FINISH
session.add(asyncrequest)
session.flush()
return
session.add(asyncrequest)
session.flush()
LOG.debug('Try cast rpc method %s' % rpc_method)
try:
rpc.cast(target, ctxt=rpc_ctxt, msg={'method': rpc_method, 'args': rpc_args})
except AMQPDestinationNotFound:
asyncrequest.resultcode = manager_common.SEND_FAIL
asyncrequest.result = 'Async %s request send fail, AMQPDestinationNotFound' % rpc_method
asyncrequest.status = manager_common.FINISH
session.flush()
return
LOG.debug('Cast %s to %s success' % (asyncrequest.request_id, target.to_dict()))
if after_run:
try:
after_run.run(asyncrequest, wait_agents)
except RpcServerCtxtException as e:
asyncrequest.result = 'Async request %s cast success, ' \
'ctxt after function error~%s' % (rpc_method, e.message)
else:
asyncrequest.result = 'Async request %s cast success' % rpc_method
finally:
session.flush()
request_id = asyncrequest.request_id
finishtime = asyncrequest.finishtime
deadline = asyncrequest.deadline + 1
expire = asyncrequest.expire
if expire:
storage = get_cache()
else:
storage = session
def check_respone():
wait = finishtime - int(time.time())
# 先等待3秒,可以做一次提前检查
if wait > 3:
eventlet.sleep(3)
no_response_agents = set(wait_agents)
interval = int(wait / 10)
if interval < 3:
interval = 3
elif interval > 10:
interval = 10
not_overtime = 2
while True:
no_response_agents = responeutils.norespones(storage=storage,
request_id=request_id,
agents=no_response_agents)
if not no_response_agents:
break
if int(time.time()) < finishtime:
eventlet.sleep(interval)
if int(time.time()) > deadline:
not_overtime -= 1
if not not_overtime:
break
eventlet.sleep(1)
LOG.debug('Not response agents count %d' % len(no_response_agents))
bulk_data = []
agent_time = int(time.time())
for agent_id in no_response_agents:
data = dict(request_id=request_id,
agent_id=agent_id,
agent_time=agent_time,
server_time=agent_time,
resultcode=manager_common.RESULT_OVER_FINISHTIME,
result='Agent respone overtime')
bulk_data.append(data)
responeutils.bluk_insert(storage, no_response_agents, bulk_data, expire)
asyncrequest.status = manager_common.FINISH
if no_response_agents:
asyncrequest.resultcode = manager_common.RESULT_NOT_ALL_SUCCESS
asyncrequest.result = 'agents not respone, count:%d' % len(no_response_agents)
else:
asyncrequest.resultcode = manager_common.RESULT_SUCCESS
asyncrequest.result = 'all agent respone result'
session.flush()
if post_run:
try:
post_run.run(asyncrequest, no_response_agents)
except RpcServerCtxtException as e:
asyncrequest.result += (' ctxt post function error~%s' % e.message)
session.flush()
session.close()
threadpool.add_thread(safe_func_wrapper, check_respone, LOG)
