def entitys(self, req, body=None):
"""批量查询entitys信息接口,内部接口agent启动的时调用,一般由agent端调用"""
entitys = body.get('entitys')
if not entitys:
return resultutils.results(result='not any app entitys found')
entitys = argutils.map_to_int(entitys)
session = endpoint_session(readonly=True)
query = model_query(session,
AppEntity,
filter=AppEntity.entity.in_(entitys))
query = query.options(joinedload(AppEntity.areas, innerjoin=False))
return resultutils.results(result='get app entitys success',
data=[
dict(entity=_entity.entity,
group_id=_entity.group_id,
status=_entity.status,
opentime=_entity.opentime,
areas=[
dict(area_id=area.area_id,
show_id=area.show_id,
areaname=area.areaname)
for area in _entity.areas
],
objtype=_entity.objtype)
for _entity in query
])
def _lock_entitys(self, endpoint, entitys):
entitys = argutils.map_to_int(entitys)
overtime = self.alloctime + int(time.time()*1000)
endpoint_key = self.ENDPOINT_KEY % endpoint
client = self.client
session = self.session(readonly=True)
query = model_query(session, AgentEntity,
filter=and_(AgentEntity.entity.in_(entitys),
AgentEntity.endpoint == endpoint))
agents_ids = set()
entitys_ids = set()
while True:
wpipe = None
try:
target_entitys = query.all()
if len(entitys) != len(target_entitys):
raise exceptions.TargetCountUnequal('Target entitys not found in database')
for entity in target_entitys:
entitys_ids.add(entity.entity)
agents_ids.add(entity.agent_id)
while int(time.time()*1000) < overtime:
with client.pipeline() as pipe:
pipe.multi()
for _id in agents_ids:
pipe.sismember(self.AGENT_KEY, str(_id))
for _id in entitys_ids:
pipe.sismember(endpoint_key, str(_id))
results = pipe.execute()
if all([True if not result else False for result in results]):
break
else:
eventlet.sleep(0.01)
wpipe = client.pipeline()
wpipe.watch(self.AGENT_KEY, endpoint_key)
wpipe.multi()
wpipe.sadd(self.AGENT_KEY, *map(str, agents_ids))
wpipe.sadd(endpoint_key, *map(str, entitys_ids))
wpipe.execute()
break
except WatchError:
if int(time.time()*1000) > overtime:
raise exceptions.AllocLockTimeout('Lock entitys timeout')
except ResponseError as e:
if not e.message.startswith('WRONGTYPE'):
raise
if int(time.time()*1000) > overtime:
raise exceptions.AllocLockTimeout('Lock entitys timeout')
finally:
if wpipe:
wpipe.reset()
try:
yield agents_ids
finally:
self.garbage_member_collection(self.AGENT_KEY, agents_ids)
self.garbage_member_collection(endpoint_key, entitys_ids)
def areas(self, req, group_id, body=None):
body = body or {}
need_ok = body.get('need_ok', False)
packages = body.get('packages', False)
group_ids = None
if group_id != 'all':
group_ids = argutils.map_to_int(group_id)
chiefs, areas = self.entitys([common.GAMESERVER, common.GMSERVER],
group_ids, need_ok, packages)
return resultutils.results(result='list group areas success',
data=[dict(chiefs=chiefs, areas=areas)])
def chiefs(self, req, group_id, body=None):
body = body or {}
cross = body.get('cross', True)
group_ids = None
if group_id != 'all':
group_ids = argutils.map_to_int(group_id)
objtypes = [common.GMSERVER]
if cross:
objtypes.append(common.CROSSSERVER)
chiefs, areas = self.entitys(objtypes, group_ids)
return resultutils.results(result='get group chiefs success',
data=chiefs)
def agents_id_check(agents_id):
global_data = get_global()
if agents_id == 'all':
return global_data.all_agents
agents_set = argutils.map_to_int(agents_id)
all_id = global_data.all_agents
if agents_set != all_id:
errors = agents_set - all_id
if errors:
raise InvalidArgument('agents id %s can not be found' %
str(list(errors)))
return agents_set
def shows(endpoint, entitys=None, agents=None, ports=True, metadata=True):
entitys_map = {}
endpoint = validateutils.validate_endpoint(endpoint)
session = get_session(readonly=True)
filters = [AgentEntity.endpoint == endpoint]
if not entitys and not agents:
return entitys_map
if entitys:
entitys = argutils.map_to_int(entitys)
filters.append(AgentEntity.entity.in_(entitys))
if agents:
agents = argutils.map_to_int(agents)
filters.append(AgentEntity.agent_id.in_(agents))
if len(filters) > 1:
filter = and_(*filters)
else:
filter = filters[0]
query = model_query(session, AgentEntity, filter=filter)
if ports:
query = query.options(
joinedload(AgentEntity.ports, innerjoin=False))
agents = set()
for _entity in query:
agents.add(_entity.agent_id)
entitys_map[_entity.entity] = dict(agent_id=_entity.agent_id)
if ports:
entitys_map[_entity.entity].setdefault(
'ports', sorted([x.port for x in _entity.ports]))
if metadata:
agents_map = BaseContorller.agents_metadata(agents)
for _entity in entitys_map:
agent_id = entitys_map[_entity].get('agent_id')
if metadata:
entitys_map[_entity].setdefault('metadata',
agents_map[agent_id])
return entitys_map
def quotes(self, req, body=None):
body = body or {}
session = endpoint_session(readonly=True)
endpoint = body.pop('endpoint')
entitys = argutils.map_to_int(body.pop('entitys'))
if len(entitys) > 5:
raise InvalidArgument('This api can not get entitys more then 5')
query = session.query(
SchemaQuote.quote_id,
SchemaQuote.schema_id,
SchemaQuote.qdatabase_id,
SchemaQuote.entity,
SchemaQuote.endpoint,
GopSchema.database_id,
GopSchema.schema,
GopSchema.user,
GopSchema.passwd,
GopSchema.ro_user,
GopSchema.ro_passwd,
GopSchema.character_set,
GopSchema.collation_type,
).join(GopSchema, and_(GopSchema.schema_id == SchemaQuote.schema_id))
query = query.filter(
and_(SchemaQuote.endpoint == endpoint,
SchemaQuote.entity.in_(entitys)))
quotes = []
database_ids = set()
for quote in query:
database_ids.add(quote.qdatabase_id)
quotes.append(
dict(quote_id=quote.quote_id,
schema_id=quote.schema_id,
qdatabase_id=quote.qdatabase_id,
database_id=quote.database_id,
schema=quote.schema,
user=quote.user,
passwd=quote.passwd,
ro_user=quote.ro_user,
ro_passwd=quote.ro_passwd,
character_set=quote.character_set,
collation_type=quote.collation_type))
# get all address
alladdress = _address(database_ids)
for quote in quotes:
quote.update(alladdress[quote.get('qdatabase_id')])
return resultutils.results(result='list quotes success', data=quotes)
def _lock_agents(self, agents):
agents = argutils.map_to_int(agents)
overtime = self.alloctime + int(time.time()*1000)
client = self.client
agents_ids = argutils.map_with(agents, str)
while True:
wpipe = None
try:
if (agents - self.all_agents):
raise exceptions.TargetCountUnequal('Target agents not in all agents id list')
locked = True
while int(time.time()*1000) < overtime:
try:
for _id in agents_ids:
if client.sismember(self.AGENT_KEY, _id):
eventlet.sleep(0.01)
continue
locked = False
except ResponseError as e:
if not e.message.startswith('WRONGTYPE'):
raise
if locked:
raise exceptions.AllocLockTimeout('Lock agents timeout')
wpipe = client.pipeline()
wpipe.watch(self.AGENT_KEY)
wpipe.multi()
wpipe.sadd(self.AGENT_KEY, *agents_ids)
wpipe.execute()
break
except WatchError:
if int(time.time()*1000) > overtime:
raise exceptions.AllocLockTimeout('Lock agents timeout')
finally:
if wpipe:
wpipe.reset()
try:
yield
finally:
if agents_ids:
self.garbage_member_collection(self.AGENT_KEY, agents_ids)
def arealock(self, group, areas):
areas = argutils.map_to_int(areas)
overtime = self.alloctime + int(time.time() * 1000)
key = self.GAGAME_GROUP_KEY % group
client = self.client
while True:
wpipe = None
try:
while int(time.time() * 1000) < overtime:
with client.pipeline() as pipe:
for _id in areas:
pipe.sismember(key, str(_id))
results = pipe.execute()
if all(
[True if not result else False for result in results]):
break
else:
eventlet.sleep(0.01)
wpipe = client.pipeline()
wpipe.watch(key)
wpipe.multi()
wpipe.sadd(key, *map(str, areas))
wpipe.execute()
break
except WatchError:
if int(time.time() * 1000) > overtime:
raise AllocLockTimeout('Lock areas timeout')
except ResponseError as e:
if not e.message.startswith('WRONGTYPE'):
if int(time.time() * 1000) > overtime:
raise AllocLockTimeout('Lock areas timeout')
raise
finally:
if wpipe:
wpipe.reset()
try:
yield areas
finally:
self.garbage_member_collection(key, map(str, areas))
def entitys(objtypes=None, group_ids=None, need_ok=False, packages=False):
filters = [
AppEntity.objtype.in_(objtypes), AppEntity.status > common.DELETED
]
if group_ids:
filters.append(
AppEntity.group_id.in_(argutils.map_to_int(group_ids)))
session = endpoint_session(readonly=True)
query = model_query(session, AppEntity, filter=and_(*filters))
query = query.options(joinedload(AppEntity.areas))
appentitys = query.all()
entitys = set()
for entity in appentitys:
entitys.add(entity.entity)
if not entitys:
return [], []
# 反查渠道
if packages and common.GAMESERVER in objtypes:
pmaps = {}
pquery = model_query(session, Package)
pquery = pquery.options(joinedload(Package.areas, innerjoin=False))
if group_ids:
pquery = pquery.filter(
Package.group_id.in_(argutils.map_to_int(group_ids)))
def _pmaps():
for package in pquery:
for parea in package.areas:
try:
pmaps[parea.area_id].append(package.package_name)
except KeyError:
pmaps[parea.area_id] = [
package.package_name,
]
th = eventlet.spawn(_pmaps)
emaps = entity_controller.shows(common.NAME,
entitys,
ports=True,
metadata=True)
if packages and common.GAMESERVER in objtypes:
th.wait()
chiefs = []
areas = []
for entity in appentitys:
if need_ok and entity.status != common.OK:
continue
entityinfo = emaps.get(entity.entity)
ports = entityinfo.get('ports')
metadata = entityinfo.get('metadata')
if not metadata:
raise ValueError('Can not get agent metadata for %d' %
entity.entity)
if entity.objtype == common.GAMESERVER:
for area in entity.areas:
info = dict(
area_id=area.area_id,
show_id=area.show_id,
areaname=area.areaname,
entity=entity.entity,
group_id=entity.group_id,
opentime=entity.opentime,
platform=entity.platform,
status=entity.status,
versions=jsonutils.loads_as_bytes(entity.versions)
if entity.versions else None,
external_ips=metadata.get('external_ips'),
dnsnames=metadata.get('dnsnames'),
port=ports[0])
if packages:
info.setdefault('packagenames',
pmaps.get(area.area_id, []))
areas.append(info)
else:
chiefs.append(
dict(entity=entity.entity,
objtype=entity.objtype,
group_id=entity.group_id,
ports=ports,
local_ip=metadata.get('local_ip'),
dnsnames=metadata.get('dnsnames'),
external_ips=metadata.get('external_ips')))
return chiefs, areas
def _async_bluck_rpc(self,
action,
group_id,
objtype,
entity,
body=None,
context=None):
caller = inspect.stack()[0][3]
body = body or {}
group_id = int(group_id)
context = context or empty_context
if entity == 'all':
entitys = 'all'
else:
entitys = argutils.map_to_int(entity)
asyncrequest = self.create_asyncrequest(body)
target = targetutils.target_endpoint(common.NAME)
session = endpoint_session(readonly=True)
query = model_query(session,
AppEntity,
filter=and_(AppEntity.group_id == group_id,
AppEntity.objtype == objtype))
emaps = dict()
for _entity in query:
if _entity.status <= common.DELETED:
continue
if _entity.status != common.OK and action != 'stop':
continue
emaps.setdefault(_entity.entity, _entity.agent_id)
if entitys == 'all':
entitys = emaps.keys()
agents = set(emaps.values())
else:
if entitys - set(emaps.keys()):
raise InvalidArgument(
'Some entitys not found or status is not active')
agents = set()
for entity in emaps:
if entity in entitys:
agents.add(emaps[entity])
with context(asyncrequest.request_id, entitys, agents):
async_ctxt = dict(pre_run=body.pop('pre_run', None),
after_run=body.pop('after_run', None),
post_run=body.pop('post_run', None))
rpc_ctxt = {}
rpc_ctxt.setdefault('agents', agents)
rpc_method = '%s_entitys' % action
rpc_args = dict(entitys=list(entitys))
rpc_args.update(body)
def wapper():
self.send_asyncrequest(asyncrequest, target, rpc_ctxt,
rpc_method, rpc_args, async_ctxt)
threadpool.add_thread(safe_func_wrapper, wapper, LOG)
return resultutils.results(
result='gogamechen2 %s entitys %s spawning' % (objtype, caller),
data=[asyncrequest.to_dict()])
