def entitys(self, req, body=None): """批量查询entitys信息接口,内部接口agent启动的时调用,一般由agent端调用""" entitys = body.get('entitys') if not entitys: return resultutils.results(result='not any app entitys found') entitys = argutils.map_to_int(entitys) session = endpoint_session(readonly=True) query = model_query(session, AppEntity, filter=AppEntity.entity.in_(entitys)) query = query.options(joinedload(AppEntity.areas, innerjoin=False)) return resultutils.results(result='get app entitys success', data=[ dict(entity=_entity.entity, group_id=_entity.group_id, status=_entity.status, opentime=_entity.opentime, areas=[ dict(area_id=area.area_id, show_id=area.show_id, areaname=area.areaname) for area in _entity.areas ], objtype=_entity.objtype) for _entity in query ])
def _lock_entitys(self, endpoint, entitys): entitys = argutils.map_to_int(entitys) overtime = self.alloctime + int(time.time()*1000) endpoint_key = self.ENDPOINT_KEY % endpoint client = self.client session = self.session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.entity.in_(entitys), AgentEntity.endpoint == endpoint)) agents_ids = set() entitys_ids = set() while True: wpipe = None try: target_entitys = query.all() if len(entitys) != len(target_entitys): raise exceptions.TargetCountUnequal('Target entitys not found in database') for entity in target_entitys: entitys_ids.add(entity.entity) agents_ids.add(entity.agent_id) while int(time.time()*1000) < overtime: with client.pipeline() as pipe: pipe.multi() for _id in agents_ids: pipe.sismember(self.AGENT_KEY, str(_id)) for _id in entitys_ids: pipe.sismember(endpoint_key, str(_id)) results = pipe.execute() if all([True if not result else False for result in results]): break else: eventlet.sleep(0.01) wpipe = client.pipeline() wpipe.watch(self.AGENT_KEY, endpoint_key) wpipe.multi() wpipe.sadd(self.AGENT_KEY, *map(str, agents_ids)) wpipe.sadd(endpoint_key, *map(str, entitys_ids)) wpipe.execute() break except WatchError: if int(time.time()*1000) > overtime: raise exceptions.AllocLockTimeout('Lock entitys timeout') except ResponseError as e: if not e.message.startswith('WRONGTYPE'): raise if int(time.time()*1000) > overtime: raise exceptions.AllocLockTimeout('Lock entitys timeout') finally: if wpipe: wpipe.reset() try: yield agents_ids finally: self.garbage_member_collection(self.AGENT_KEY, agents_ids) self.garbage_member_collection(endpoint_key, entitys_ids)
def areas(self, req, group_id, body=None): body = body or {} need_ok = body.get('need_ok', False) packages = body.get('packages', False) group_ids = None if group_id != 'all': group_ids = argutils.map_to_int(group_id) chiefs, areas = self.entitys([common.GAMESERVER, common.GMSERVER], group_ids, need_ok, packages) return resultutils.results(result='list group areas success', data=[dict(chiefs=chiefs, areas=areas)])
def chiefs(self, req, group_id, body=None): body = body or {} cross = body.get('cross', True) group_ids = None if group_id != 'all': group_ids = argutils.map_to_int(group_id) objtypes = [common.GMSERVER] if cross: objtypes.append(common.CROSSSERVER) chiefs, areas = self.entitys(objtypes, group_ids) return resultutils.results(result='get group chiefs success', data=chiefs)
def agents_id_check(agents_id): global_data = get_global() if agents_id == 'all': return global_data.all_agents agents_set = argutils.map_to_int(agents_id) all_id = global_data.all_agents if agents_set != all_id: errors = agents_set - all_id if errors: raise InvalidArgument('agents id %s can not be found' % str(list(errors))) return agents_set
def shows(endpoint, entitys=None, agents=None, ports=True, metadata=True): entitys_map = {} endpoint = validateutils.validate_endpoint(endpoint) session = get_session(readonly=True) filters = [AgentEntity.endpoint == endpoint] if not entitys and not agents: return entitys_map if entitys: entitys = argutils.map_to_int(entitys) filters.append(AgentEntity.entity.in_(entitys)) if agents: agents = argutils.map_to_int(agents) filters.append(AgentEntity.agent_id.in_(agents)) if len(filters) > 1: filter = and_(*filters) else: filter = filters[0] query = model_query(session, AgentEntity, filter=filter) if ports: query = query.options( joinedload(AgentEntity.ports, innerjoin=False)) agents = set() for _entity in query: agents.add(_entity.agent_id) entitys_map[_entity.entity] = dict(agent_id=_entity.agent_id) if ports: entitys_map[_entity.entity].setdefault( 'ports', sorted([x.port for x in _entity.ports])) if metadata: agents_map = BaseContorller.agents_metadata(agents) for _entity in entitys_map: agent_id = entitys_map[_entity].get('agent_id') if metadata: entitys_map[_entity].setdefault('metadata', agents_map[agent_id]) return entitys_map
def quotes(self, req, body=None): body = body or {} session = endpoint_session(readonly=True) endpoint = body.pop('endpoint') entitys = argutils.map_to_int(body.pop('entitys')) if len(entitys) > 5: raise InvalidArgument('This api can not get entitys more then 5') query = session.query( SchemaQuote.quote_id, SchemaQuote.schema_id, SchemaQuote.qdatabase_id, SchemaQuote.entity, SchemaQuote.endpoint, GopSchema.database_id, GopSchema.schema, GopSchema.user, GopSchema.passwd, GopSchema.ro_user, GopSchema.ro_passwd, GopSchema.character_set, GopSchema.collation_type, ).join(GopSchema, and_(GopSchema.schema_id == SchemaQuote.schema_id)) query = query.filter( and_(SchemaQuote.endpoint == endpoint, SchemaQuote.entity.in_(entitys))) quotes = [] database_ids = set() for quote in query: database_ids.add(quote.qdatabase_id) quotes.append( dict(quote_id=quote.quote_id, schema_id=quote.schema_id, qdatabase_id=quote.qdatabase_id, database_id=quote.database_id, schema=quote.schema, user=quote.user, passwd=quote.passwd, ro_user=quote.ro_user, ro_passwd=quote.ro_passwd, character_set=quote.character_set, collation_type=quote.collation_type)) # get all address alladdress = _address(database_ids) for quote in quotes: quote.update(alladdress[quote.get('qdatabase_id')]) return resultutils.results(result='list quotes success', data=quotes)
def _lock_agents(self, agents): agents = argutils.map_to_int(agents) overtime = self.alloctime + int(time.time()*1000) client = self.client agents_ids = argutils.map_with(agents, str) while True: wpipe = None try: if (agents - self.all_agents): raise exceptions.TargetCountUnequal('Target agents not in all agents id list') locked = True while int(time.time()*1000) < overtime: try: for _id in agents_ids: if client.sismember(self.AGENT_KEY, _id): eventlet.sleep(0.01) continue locked = False except ResponseError as e: if not e.message.startswith('WRONGTYPE'): raise if locked: raise exceptions.AllocLockTimeout('Lock agents timeout') wpipe = client.pipeline() wpipe.watch(self.AGENT_KEY) wpipe.multi() wpipe.sadd(self.AGENT_KEY, *agents_ids) wpipe.execute() break except WatchError: if int(time.time()*1000) > overtime: raise exceptions.AllocLockTimeout('Lock agents timeout') finally: if wpipe: wpipe.reset() try: yield finally: if agents_ids: self.garbage_member_collection(self.AGENT_KEY, agents_ids)
def arealock(self, group, areas): areas = argutils.map_to_int(areas) overtime = self.alloctime + int(time.time() * 1000) key = self.GAGAME_GROUP_KEY % group client = self.client while True: wpipe = None try: while int(time.time() * 1000) < overtime: with client.pipeline() as pipe: for _id in areas: pipe.sismember(key, str(_id)) results = pipe.execute() if all( [True if not result else False for result in results]): break else: eventlet.sleep(0.01) wpipe = client.pipeline() wpipe.watch(key) wpipe.multi() wpipe.sadd(key, *map(str, areas)) wpipe.execute() break except WatchError: if int(time.time() * 1000) > overtime: raise AllocLockTimeout('Lock areas timeout') except ResponseError as e: if not e.message.startswith('WRONGTYPE'): if int(time.time() * 1000) > overtime: raise AllocLockTimeout('Lock areas timeout') raise finally: if wpipe: wpipe.reset() try: yield areas finally: self.garbage_member_collection(key, map(str, areas))
def entitys(objtypes=None, group_ids=None, need_ok=False, packages=False): filters = [ AppEntity.objtype.in_(objtypes), AppEntity.status > common.DELETED ] if group_ids: filters.append( AppEntity.group_id.in_(argutils.map_to_int(group_ids))) session = endpoint_session(readonly=True) query = model_query(session, AppEntity, filter=and_(*filters)) query = query.options(joinedload(AppEntity.areas)) appentitys = query.all() entitys = set() for entity in appentitys: entitys.add(entity.entity) if not entitys: return [], [] # 反查渠道 if packages and common.GAMESERVER in objtypes: pmaps = {} pquery = model_query(session, Package) pquery = pquery.options(joinedload(Package.areas, innerjoin=False)) if group_ids: pquery = pquery.filter( Package.group_id.in_(argutils.map_to_int(group_ids))) def _pmaps(): for package in pquery: for parea in package.areas: try: pmaps[parea.area_id].append(package.package_name) except KeyError: pmaps[parea.area_id] = [ package.package_name, ] th = eventlet.spawn(_pmaps) emaps = entity_controller.shows(common.NAME, entitys, ports=True, metadata=True) if packages and common.GAMESERVER in objtypes: th.wait() chiefs = [] areas = [] for entity in appentitys: if need_ok and entity.status != common.OK: continue entityinfo = emaps.get(entity.entity) ports = entityinfo.get('ports') metadata = entityinfo.get('metadata') if not metadata: raise ValueError('Can not get agent metadata for %d' % entity.entity) if entity.objtype == common.GAMESERVER: for area in entity.areas: info = dict( area_id=area.area_id, show_id=area.show_id, areaname=area.areaname, entity=entity.entity, group_id=entity.group_id, opentime=entity.opentime, platform=entity.platform, status=entity.status, versions=jsonutils.loads_as_bytes(entity.versions) if entity.versions else None, external_ips=metadata.get('external_ips'), dnsnames=metadata.get('dnsnames'), port=ports[0]) if packages: info.setdefault('packagenames', pmaps.get(area.area_id, [])) areas.append(info) else: chiefs.append( dict(entity=entity.entity, objtype=entity.objtype, group_id=entity.group_id, ports=ports, local_ip=metadata.get('local_ip'), dnsnames=metadata.get('dnsnames'), external_ips=metadata.get('external_ips'))) return chiefs, areas
def _async_bluck_rpc(self, action, group_id, objtype, entity, body=None, context=None): caller = inspect.stack()[0][3] body = body or {} group_id = int(group_id) context = context or empty_context if entity == 'all': entitys = 'all' else: entitys = argutils.map_to_int(entity) asyncrequest = self.create_asyncrequest(body) target = targetutils.target_endpoint(common.NAME) session = endpoint_session(readonly=True) query = model_query(session, AppEntity, filter=and_(AppEntity.group_id == group_id, AppEntity.objtype == objtype)) emaps = dict() for _entity in query: if _entity.status <= common.DELETED: continue if _entity.status != common.OK and action != 'stop': continue emaps.setdefault(_entity.entity, _entity.agent_id) if entitys == 'all': entitys = emaps.keys() agents = set(emaps.values()) else: if entitys - set(emaps.keys()): raise InvalidArgument( 'Some entitys not found or status is not active') agents = set() for entity in emaps: if entity in entitys: agents.add(emaps[entity]) with context(asyncrequest.request_id, entitys, agents): async_ctxt = dict(pre_run=body.pop('pre_run', None), after_run=body.pop('after_run', None), post_run=body.pop('post_run', None)) rpc_ctxt = {} rpc_ctxt.setdefault('agents', agents) rpc_method = '%s_entitys' % action rpc_args = dict(entitys=list(entitys)) rpc_args.update(body) def wapper(): self.send_asyncrequest(asyncrequest, target, rpc_ctxt, rpc_method, rpc_args, async_ctxt) threadpool.add_thread(safe_func_wrapper, wapper, LOG) return resultutils.results( result='gogamechen2 %s entitys %s spawning' % (objtype, caller), data=[asyncrequest.to_dict()])