def delete(self, req, agent_id, endpoint): endpoints = validateutils.validate_endpoints(endpoint) if not endpoints: raise InvalidArgument('Endpoints is None for add endpoints') endpoints = validateutils.validate_endpoints(endpoints) session = get_session() glock = get_global().lock('agents') with glock([ agent_id, ]): with session.begin(): if model_count_with_key( session, AgentEntity.entity, filter=and_(AgentEntity.agent_id == agent_id, AgentEntity.endpoint.in_(endpoints))): return resultutils.results( resultcode=manager_common.RESULT_ERROR, result= 'delete endpoints fail, entitys still in endpoint') query = model_query(session, AgentEndpoint, filter=and_( AgentEndpoint.agent_id == agent_id, AgentEndpoint.endpoint.in_(endpoints))) delete_count = query.delete(synchronize_session=False) need_to_delete = len(endpoints) if delete_count != len(endpoints): LOG.warning('Delete %d endpoints, but expect count is %d' % (delete_count, need_to_delete)) session.close() return resultutils.results(result='delete endpoints success')
def create_database(self, user, passwd, dbtype, dbversion, affinity, **kwargs): """create new database intance""" session = endpoint_session() with session.begin(): bond = kwargs.pop('bond', None) if bond: query = model_query(session, GopDatabase, filter=GopDatabase.database_id == bond) bond = query.one_or_none() if not bond: raise InvalidArgument( 'Target slave databse can not be found, can not bond to slave databases' ) if not bond.slave: raise InvalidArgument( 'Target database is not salve database') if bond.status != common.OK: raise InvalidArgument('Targe slave database is not active') count = model_count_with_key( session, GopSalveRelation, filter=GopSalveRelation.slave_id == bond.database_id) if count >= bond.slave: raise InvalidArgument('Target slave database is full') else: bond = None _database = GopDatabase(user=user, passwd=passwd, slave=kwargs.pop('slave'), dbtype=dbtype, dbversion=dbversion, affinity=affinity, desc=kwargs.pop('desc', None)) _result = dict(dbversion=_database.dbversion, slave=_database.slave, dbtype=_database.dbtype) with self._create_database(session, _database, bond, **kwargs) as address: host = address[0] port = address[1] _result.setdefault('host', host) _result.setdefault('port', port) _result.setdefault('affinity', affinity) session.add(_database) session.flush() if bond: LOG.info('Add GopSalveRelation for database') relation = GopSalveRelation( master_id=_database.database_id, slave_id=bond.database_id) session.add(relation) session.flush() self._esure_create(_database, **kwargs) _result.setdefault('database_id', _database.database_id) return _result
def quote_version(self, req, group_id, objtype, entity, body=None): """区服包引用指定资源版本""" body = body or {} if objtype != common.GAMESERVER: raise InvalidArgument('Version quote just for %s' % common.GAMESERVER) package_id = int(body.get('package_id')) rversion = body.get('rversion') group_id = int(group_id) entity = int(entity) session = endpoint_session() query = model_query(session, Group, filter=Group.group_id == group_id) query = query.options(joinedload(Group.packages, innerjoin=False)) group = query.one() resource_id = None for package in group.packages: if package.package_id == package_id: resource_id = package.resource_id if not resource_id: raise InvalidArgument('Entity can not find package or package resource is None') query = model_query(session, AppEntity, filter=AppEntity.entity == entity) query = query.options(joinedload(AppEntity.areas, innerjoin=False)) with session.begin(): _entity = query.one() if _entity.objtype != objtype: raise InvalidArgument('Objtype not match') if _entity.group_id != group_id: raise InvalidArgument('Group id not match') if not model_count_with_key(session, PackageArea.package_id, filter=and_(PackageArea.package_id == package_id, PackageArea.area_id.in_([area.area_id for area in _entity.areas]) )): raise InvalidArgument('Entity area not in package areas') versions = jsonutils.loads_as_bytes(_entity.versions) if _entity.versions else {} str_key = str(package_id) if str_key in versions: quote = versions.get(str_key) if quote.get('version') != rversion: body = {'version': rversion} quote.update(body) cdnquote_controller.update(req, quote.get('quote_id'), body=body) else: qresult = cdnresource_controller.vquote(req, resource_id, body={'version': rversion, 'desc': '%s.%d' % (common.NAME, entity)}) quote = qresult['data'][0] quote = dict(version=rversion, quote_id=quote.get('quote_id')) versions.setdefault(str_key, quote) _entity.versions = jsonutils.dumps(versions) session.flush() return resultutils.results(result='set entity version quote success', data=[dict(resource_id=resource_id, version=rversion, quote_id=quote.get('quote_id'))])
def count(self, req, endpoint): session = get_session(readonly=True) data = [] for endpoint in argutils.map_with(endpoint, validateutils.validate_endpoint): count = model_count_with_key( session, AgentEndpoint.endpoint, filter=AgentEndpoint.endpoint == endpoint) data.append(dict(endpoint=endpoint, count=count)) return resultutils.results(result='count endpoint for success', data=data)
def slave_database(self, database_id, **kwargs): slave_id = kwargs.pop('slave') file = kwargs.get('file') position = kwargs.get('position') master = None slave = None session = endpoint_session() query = model_query(session, GopDatabase, filter=GopDatabase.database_id.in_( [database_id, slave_id])) query = query.options(joinedload(GopDatabase.slaves, innerjoin=False)) with session.begin(subtransactions=True): for database in query: if database.database_id == database_id: master = database elif database.database_id == slave_id: slave = database # 校验主从 if slave is None or slave.slave <= 0: raise InvalidArgument( 'slave database with id %d can not be found' % slave_id) if slave.status != common.OK: raise InvalidArgument('Slave database is not active') if master is None or master.slave > 0: raise InvalidArgument( 'Master database with id %d can not be found' % database_id) if master.impl != slave.impl or master.dbtype != slave.dbtype: raise InvalidArgument( 'Master and slave not the same type or impl') schemas = [schema.schema for schema in master.schemas] # master中有schemas if schemas: if not file or not position: raise InvalidArgument( 'Can not bond slave without file and position') for _relation in master.slaves: # 找到绑定关系 if _relation.slave_id == slave_id: raise InvalidArgument('Slave already in master slave list') # 没有绑定关系, 确认从库上限 count = model_count_with_key( session, GopSalveRelation, filter=GopSalveRelation.slave_id == slave_id) if count >= slave.slave: raise InvalidArgument('Target slave database is full') kwargs['schemas'] = schemas return self._slave_database(session, master, slave, **kwargs)
def add_agent(self, body): agent = Agent() try: agent.host = validators['type:hostname'](body.pop('host')) agent.agent_type = body.pop('agent_type', None) if agent.agent_type is None or len(agent.agent_type) > 64: raise ValueError('Agent type info over size') if body.get('ports_range', None) is not None: agent.ports_range = jsonutils.dumps(body.pop('ports_range')) agent.memory = int(body.pop('memory')) agent.cpu = int(body.pop('cpu')) agent.disk = int(body.pop('disk')) endpoints = validateutils.validate_endpoints(body.pop('endpoints', [])) except KeyError as e: raise InvalidArgument('Can not find argument: %s' % e.message) except ValueError as e: raise InvalidArgument('Argument value type error: %s' % e.message) agent.create_time = timeutils.realnow() session = self.session() with self._lock_all_agents(): host_filter = and_(Agent.host == agent.host, Agent.status > manager_common.DELETED) if model_count_with_key(session, Agent.host, filter=host_filter) > 0: raise exceptions.AgentHostExist('Agent with host %s alreday eixst' % agent.host) agent_id = model_autoincrement_id(session, Agent.agent_id) with session.begin(): agent.agent_id = agent_id # if agent.endpoints: # for endpoint in agent.endpoints: # endpoint.agent_id = agent_id session.add(agent) session.flush() if endpoints: for endpoint in endpoints: session.add(AgentEndpoint(agent_id=agent_id, endpoint=endpoint)) session.flush() # add new agent_id to cache all agent_id # if not self.client.sadd(self.ALL_AGENTS_KEY, str(agent.agent_id)): if not self.client.zadd(self.ALL_AGENTS_KEY, int(time.time()), str(agent.agent_id)): raise exceptions.CacheStoneError('Cant not add agent_id to redis, key %s' % self.ALL_AGENTS_KEY) _endpoints = [e.endpoint for e in agent.endpoints] LOG.info('New anget with endpoints %s' % ','.join(_endpoints)) return agent
def mark(self, req, cid, uid, body=None): """收藏漫画""" cid = int(cid) uid = int(uid) session = endpoint_session() if model_count_with_key(session, UserBook, filter=UserBook.uid == uid) >= common.MAXBOOKS: raise InvalidArgument('Mark over 50') query = model_query(session, Comic.name, filter=Comic.cid == cid) comic = query.one() try: session.add( UserBook(uid=uid, cid=cid, ext=comic.ext, name=comic.name, time=int(time.time()))) session.flush() except DBDuplicateEntry: LOG.warning('User alreday mark comic') return resultutils.results(result='mark book success', data=[dict(cid=comic.cid, name=comic.name)])
def area(self, req, group_id, body=None): """change entity area""" body = body or {} try: group_id = int(group_id) except (TypeError, ValueError): raise InvalidArgument('Group id value error') area_id = body.get('area_id') areaname = body.get('areaname') show_id = body.get('show_id') if not areaname and not show_id: raise InvalidArgument('No value change') rpc = get_client() session = endpoint_session() query = model_query(session, GameArea, filter=GameArea.area_id == area_id) with session.begin(): area = query.one_or_none() if not area: raise InvalidArgument('No area found') if area.group_id != group_id: raise InvalidArgument('Area group not %d' % group_id) entityinfo = entity_controller.show(req=req, entity=area.entity, endpoint=common.NAME, body={'ports': False})['data'][0] agent_id = entityinfo['agent_id'] metadata = entityinfo['metadata'] if not metadata: raise InvalidArgument( 'Agent is off line, can not reset entity') if areaname: if model_count_with_key(session, GameArea, filter=and_( GameArea.group_id == group_id, GameArea.areaname == areaname)): raise InvalidArgument('Area name duplicate in group %d' % group_id) area.areaname = areaname if show_id: area.show_id = show_id target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME finishtime, timeout = rpcfinishtime() rpc_ret = rpc.call(target, ctxt={ 'finishtime': finishtime, 'agents': [ agent_id, ] }, msg={ 'method': 'change_entity_area', 'args': dict(entity=area.entity, area_id=area.area_id, show_id=area.show_id, areaname=area.areaname) }, timeout=timeout) if not rpc_ret: raise RpcResultError('change entity area result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('change entity area fail %s' % rpc_ret.get('result')) session.flush() return resultutils.results(result='change group areas success')
def create(self, req, agent_id, endpoint, body=None, action='create'): body = body or {} endpoint = validateutils.validate_endpoint(endpoint) ports = body.pop('ports', None) notify = body.pop('notify', True) desc = body.pop('desc', None) session = get_session() metadata = None if ports: ports = argutils.map_with(ports, validators['type:port']) used_ports = model_count_with_key( session, AllocatedPort.port, filter=and_(AllocatedPort.agent_id == agent_id, AllocatedPort.port.in_(ports))) if used_ports: raise InvalidArgument('Ports has been used count %d' % used_ports) if notify: metadata = BaseContorller.agent_metadata(agent_id) # make sure agent is online if not metadata: raise RpcPrepareError( 'Can not create entity on a offline agent %d' % agent_id) entity = 0 glock = get_global().lock('agents') elock = get_global().lock('endpoint') result = 'add entity success.' with glock([ agent_id, ]): with elock(endpoint): with session.begin(subtransactions=True): query = model_query(session, Agent, filter=Agent.agent_id == agent_id) query = query.options( joinedload(Agent.endpoints, innerjoin=False)) agent = query.one() if agent.status != manager_common.ACTIVE: raise InvalidArgument( 'Create entity fail, agent status is not active') _endpoint = None for e in agent.endpoints: if endpoint == e.endpoint: _endpoint = e break if not _endpoint: raise InvalidArgument( 'Create entity fail, agent %d has no endpoint %s' % (agent_id, endpoint)) entity_autoincrement_id = model_autoincrement_id( session, AgentEntity.entity, filter=AgentEntity.endpoint == endpoint) entity = body.get('entity') or entity_autoincrement_id if entity < entity_autoincrement_id: raise InvalidArgument( 'Create entity fail, entity less then autoincrement id' ) _entity = AgentEntity(entity=entity, endpoint=endpoint, agent_id=agent_id, endpoint_id=_endpoint.id, desc=desc) session.add(_entity) session.flush() LOG.info('Create entity %s.%d with entity id %s' % (endpoint, entity, _entity.id)) if ports: for port in ports: session.add( AllocatedPort(port=port, agent_id=agent_id, endpoint_id=_endpoint.id, entity_id=entity.id, endpoint=endpoint, entity=entity)) session.flush() if notify: target = targetutils.target_agent(agent) target.namespace = endpoint create_result = self.notify_create( target, agent.agent_id, entity, body, action) if not create_result: raise RpcResultError( 'create entitys result is None') if create_result.get( 'resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('create entity fail %s' % create_result.get('result')) result += create_result.get('result') notify = create_result return resultutils.results(result=result, data=[ dict(entity=entity, agent_id=agent_id, metadata=metadata, endpoint=endpoint, ports=ports or [], notify=notify) ])
def delete(self, req, group_id, objtype, entity, body=None): """标记删除entity""" body = body or {} force = body.get('force', False) group_id = int(group_id) entity = int(entity) session = endpoint_session() glock = get_gamelock() metadata, ports = self._entityinfo(req=req, entity=entity) if not metadata: raise InvalidArgument('Agent offline, can not delete entity') query = model_query(session, AppEntity, filter=AppEntity.entity == entity) if objtype == common.GAMESERVER: query = query.options(joinedload(AppEntity.areas, innerjoin=False)) _entity = query.one() if _entity.status == common.DELETED: return resultutils.results(result='mark %s entity delete success' % objtype, data=[ dict(entity=entity, objtype=objtype, ports=ports, metadata=metadata) ]) if _entity.objtype != objtype: raise InvalidArgument('Objtype not match') if _entity.group_id != group_id: raise InvalidArgument('Group id not match') target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) target.namespace = common.NAME rpc = get_client() with glock.grouplock(group=group_id): if objtype == common.GMSERVER: if model_count_with_key( session, AppEntity, filter=AppEntity.group_id == group_id) > 1: raise InvalidArgument( 'You must delete other objtype entity before delete gm' ) if model_count_with_key( session, Package, filter=Package.group_id == group_id) > 1: raise InvalidArgument( 'You must delete other Package before delete gm') elif objtype == common.CROSSSERVER: if model_count_with_key( session, AppEntity, filter=AppEntity.cross_id == _entity.entity): raise InvalidArgument('Cross server are reflected') with session.begin(): # 确认实体没有运行 rpc_ret = rpc.call(target, ctxt={'agents': [ _entity.agent_id, ]}, msg={ 'method': 'stoped', 'args': dict(entity=entity) }) if not rpc_ret: raise RpcResultError('check entity stoped result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('check entity fail %s' % rpc_ret.get('result')) _entity.status = common.DELETED session.flush() if objtype == common.GAMESERVER: # 删除所有资源版本引用 if _entity.versions: for quote in six.itervalues( jsonutils.loads_as_bytes(_entity.versions)): threadpool.add_thread(cdnquote_controller.delete, req, quote.get('quote_id')) _entity.versions = None session.flush() if _entity.areas: if len(_entity.areas) > 1: raise InvalidArgument('%s areas more then one' % objtype) area = _entity.areas[0] if not force: if _entity.entity != model_max_with_key( session, AppEntity.entity, filter=and_( AppEntity.objtype == common.GAMESERVER, AppEntity.group_id == group_id)): raise InvalidArgument( 'entity %d is not the last gamesvr entity in group' % entity) session.flush() session.delete(area) session.flush() _query = model_query( session, PackageArea, filter=PackageArea.area_id == area.area_id) _query.delete() session.flush() rpc.cast(target, ctxt={'agents': [ _entity.agent_id, ]}, msg={ 'method': 'change_status', 'args': dict(entity=entity, status=common.DELETED) }) return resultutils.results(result='mark %s entity delete success' % objtype, data=[ dict(entity=entity, objtype=objtype, ports=ports, metadata=metadata) ])
def create(self, req, group_id, objtype, body=None): body = body or {} group_id = int(group_id) jsonutils.schema_validate(body, self.CREATEAPPENTITY) if body.get('packages'): raise InvalidArgument('Package parameter is removed') # 找cross服务, gameserver专用 cross_id = body.pop('cross_id', None) # 开服时间, gameserver专用 opentime = body.pop('opentime', None) # 区服显示id, gameserver专用 show_id = body.pop('show_id', None) # 区服显示民称, gameserver专用 areaname = body.pop('areaname', None) # 平台类型 platform = body.get('platform') include = set(body.pop('include', [])) exclude = set(body.pop('exclude', [])) if include and exclude: raise InvalidArgument('Both packages and exclude is forbidden') packages = [] session = endpoint_session() if objtype == common.GAMESERVER: platform = common.PlatformTypeMap.get(platform) if not areaname or not opentime or not platform or not show_id: raise InvalidArgument( '%s need opentime and areaname and platform and show_id' % objtype) # 安装文件信息 appfile = body.pop(common.APPFILE) LOG.debug('Try find agent and database for entity') # 选择实例运行服务器 agent_id = body.get('agent_id') or self._agentselect( req, objtype, **body) # 选择实例运行数据库 databases = self._dbselect(req, objtype, **body) # 校验数据库信息 if not self._validate_databases(objtype, databases): raise InvalidArgument('Miss some database') LOG.info( 'Find agent and database for entity success, to agent %d, to databse %s' % (agent_id, str(databases))) query = model_query(session, Group, filter=Group.group_id == group_id) joins = joinedload(Group.entitys, innerjoin=False) joins = joins.joinedload(AppEntity.databases, innerjoin=False) query = query.options(joins) _group = query.one() glock = get_gamelock() with glock.grouplock(group_id): if objtype == common.GAMESERVER: _pquery = model_query(session, Package, filter=Package.group_id == group_id) _packages = set([ p.package_id for p in _pquery.all() if p.platform & platform ]) if (include - _packages) or (exclude - _packages): raise InvalidArgument( 'Package can not be found in include or exclude') if exclude: packages = _packages - exclude elif include: packages = include else: packages = _packages typemap = {} for _entity in _group.entitys: # 跳过未激活的实体 if _entity.status != common.OK: continue try: typemap[_entity.objtype].append(_entity) except KeyError: typemap[_entity.objtype] = [ _entity, ] # 前置实体 chiefs = None # 相同类型的实例列表 same_type_entitys = typemap.get(objtype, []) if objtype == common.GMSERVER: # GM服务不允许相同实例,必须clean掉所有同组GM服务器 for _entity in _group.entitys: if _entity.objtype == common.GMSERVER: return resultutils.results( result='create entity fail, %s duplicate in group' % objtype, resultcode=manager_common.RESULT_ERROR) else: # 非gm实体添加需要先找到同组的gm try: gm = typemap[common.GMSERVER][0] if gm.status <= common.DELETED: return resultutils.results( result='Create entity fail, gm mark deleted', resultcode=manager_common.RESULT_ERROR) except KeyError as e: return resultutils.results( result='Create entity fail, can not find GMSERVER: %s' % e.message, resultcode=manager_common.RESULT_ERROR) if objtype == common.GAMESERVER: if model_count_with_key( session, GameArea, filter=and_(GameArea.group_id == group_id, GameArea.areaname == areaname)): return resultutils.results( result='Create entity fail, name exist', resultcode=manager_common.RESULT_ERROR) cross = None # 游戏服务器需要在同组中找到cross实例 try: crossservers = typemap[common.CROSSSERVER] except KeyError as e: return resultutils.results( result= 'create entity fail, can not find my chief: %s' % e.message, resultcode=manager_common.RESULT_ERROR) # 如果指定了cross实例id if cross_id: # 判断cross实例id是否在当前组中 for _cross in crossservers: if cross_id == _cross.entity: cross = _cross break else: # 游戏服没有相同实例,直接使用第一个cross实例 if not same_type_entitys: cross = crossservers[0] else: # 统计所有cross实例的引用次数 counted = set() counter = dict() for _cross in crossservers: counter.setdefault(_cross.entity, 0) # 查询当前组内所有entity对应的cross_id for _entity in _group.entitys: if _entity.objtype != common.GAMESERVER: continue if _entity.cross_id in counted: continue counter[_entity.cross_id] += 1 # 选取引用次数最少的cross_id cross_id = sorted( zip(counter.itervalues(), counter.iterkeys()))[0][1] for _cross in crossservers: if cross_id == _cross.entity: cross = _cross break if not cross: raise InvalidArgument( 'cross server can not be found or not active') # 获取实体相关服务器信息(端口/ip) maps = entity_controller.shows( endpoint=common.NAME, entitys=[gm.entity, cross.entity]) for v in six.itervalues(maps): if v is None: raise InvalidArgument( 'Get chiefs info error, agent not online?') chiefs = dict() # 战场与GM服务器信息 for chief in (cross, gm): chiefmetadata = maps.get(chief.entity).get('metadata') ports = maps.get(chief.entity).get('ports') if not chiefmetadata: raise InvalidArgument( '%s.%d is offline' % (chief.objtype, chief.entity)) need = common.POSTS_COUNT[chief.objtype] if need and len(ports) != need: raise InvalidArgument('%s.%d port count error, ' 'find %d, need %d' % (chief.objtype, chief.entity, len(ports), need)) chiefs.setdefault( chief.objtype, dict(entity=chief.entity, ports=ports, local_ip=chiefmetadata.get('local_ip'))) cross_id = cross.entity # 完整的rpc数据包 create_body = dict(objtype=objtype, appfile=appfile, databases=databases, chiefs=chiefs, entity=int(body.get('entity', 0))) with session.begin(): body.setdefault('finishtime', rpcfinishtime()[0] + 5) try: create_result = entity_controller.create( req=req, agent_id=agent_id, endpoint=common.NAME, body=create_body)['data'][0] except RpcResultError as e: LOG.error('Create entity rpc call fail: %s' % e.message) raise InvalidArgument(e.message) entity = create_result.get('entity') rpc_result = create_result.get('notify') LOG.info('Create new entity %d' % entity) LOG.debug('Entity controller create rpc result %s', str(rpc_result)) # 插入实体信息 appentity = AppEntity(entity=entity, agent_id=agent_id, group_id=group_id, objtype=objtype, cross_id=cross_id, opentime=opentime, platform=platform) session.add(appentity) session.flush() if objtype == common.GAMESERVER: areaname = areaname.decode('utf-8') if isinstance( areaname, six.binary_type) else areaname gamearea = GameArea(group_id=_group.group_id, show_id=show_id, areaname=areaname, gid=None, entity=appentity.entity) session.add(gamearea) session.flush() # area id插入渠道包包含列表中,批量操作 if packages: for package_id in packages: session.add( PackageArea(package_id=package_id, area_id=gamearea.area_id)) session.flush() # 插入数据库绑定信息 if rpc_result.get('databases'): self._bondto(session, entity, rpc_result.get('databases')) else: LOG.error('New entity database miss') _result = dict(entity=entity, objtype=objtype, agent_id=agent_id, connection=rpc_result.get('connection'), ports=rpc_result.get('ports'), databases=rpc_result.get('databases')) areas = [] if objtype == common.GAMESERVER: areas = [ dict(area_id=gamearea.area_id, gid=0, areaname=areaname, show_id=show_id) ] _result.setdefault('areas', areas) _result.setdefault('cross_id', cross_id) _result.setdefault('opentime', opentime) _result.setdefault('platform', platform) _result.setdefault('packages', sorted(packages)) # 添加端口 # threadpool.add_thread(port_controller.unsafe_create, # agent_id, common.NAME, entity, rpc_result.get('ports')) port_controller.unsafe_create(agent_id, common.NAME, entity, rpc_result.get('ports')) # agent 后续通知 threadpool.add_thread(entity_controller.post_create_entity, entity, common.NAME, objtype=objtype, status=common.UNACTIVE, opentime=opentime, group_id=group_id, areas=areas) return resultutils.results(result='create %s entity success' % objtype, data=[ _result, ])
def bulk_results(session, model, columns, counter=None, order=None, desc=None, filter=None, option=None, page_num=0, limit=None): query = model_query(session, model, filter=filter) if option: query = query.options(option) def validator(_column): if isinstance(_column, basestring): if hasattr(model, _column) and isinstance(getattr(_column, _column), InstrumentedAttribute): intance = model.__dict__[_column] name = _column else: raise InvalidArgument('Can not find column %s in %s' % (_column, model.__tablename__)) elif isinstance(counter, InstrumentedAttribute): if counter.class_ is not model: raise InvalidArgument('Column %s not belong to table %s' % (counter.key, model.__tablename__)) intance = _column name = _column.key else: raise InvalidArgument('Column value not basestring or intance of InstrumentedAttribute') return intance, name # formater counter if counter is None: counter = model counter = validator(counter)[0] # formater order if order is not None: order = validator(order)[0] if desc: order = order.desc() query = query.order_by(order) # format columns list to basestring if not columns: raise ValueError('Result column is None') column_name_list = set() for column in columns: try: column_name = validator(column)[1] except InvalidArgument as e: raise ValueError(e.message) column_name_list.add(column_name) # check page number if page_num: # count row number all_rows_num = model_count_with_key(session, counter, filter=filter) if page_num*manager_common.ROW_PER_PAGE >= all_rows_num: raise InvalidArgument('Page number over size or no data exist') query.seek(page_num*manager_common.ROW_PER_PAGE) limit = limit or manager_common.MAX_ROW_PER_REQUEST query = query.limit(limit) row_list = [] for result in query: column = dict() for column_name in column_name_list: column[column_name] = getattr(result, column_name) row_list.append(column) result = 'Get results success' if len(row_list) == 0: result = 'No result found' ret_dict = results(total=0, pagenum=page_num, data=row_list, result=result) return ret_dict
def bond_database(self, database_id, **kwargs): master_id = kwargs.pop('master') file = kwargs.get('file') position = kwargs.get('position') schemas = kwargs.get('schemas') if not file or not position: raise InvalidArgument( 'Can not bond slave without file and position') master = None slave = None relation = None session = endpoint_session() query = model_query(session, GopDatabase, filter=GopDatabase.database_id.in_( [database_id, master_id])) query = query.options(joinedload(GopDatabase.slaves, innerjoin=False)) with session.begin(subtransactions=True): for database in query: if database.database_id == database_id: slave = database elif database.database_id == master_id: master = database # 校验主从 if slave is None or slave.slave <= 0: raise InvalidArgument( 'slave database with id %d can not be found' % database_id) if slave.status != common.OK: raise InvalidArgument('Slave database is not active') if master is None or master.slave > 0: raise InvalidArgument( 'Master database with id %d can not be found' % master_id) if master.impl != slave.impl or master.dbtype != slave.dbtype: raise InvalidArgument( 'Master and slave not the same type or impl') _schemas = set([schema.schema for schema in master.schemas]) # 校验master中的schemas是否正确 if set(_schemas) != set(schemas): raise ValueError('Master schemas info error') for _relation in master.slaves: if _relation.slave_id == database_id: # 找到绑定关系 if _relation.ready: raise InvalidArgument( 'Slave already in master slave list') relation = _relation break # 没有绑定关系, 确认从库上限 if not relation: count = model_count_with_key( session, GopSalveRelation, filter=GopSalveRelation.slave_id == database_id) if count >= slave.slave: raise InvalidArgument('Target slave database is full') relation = GopSalveRelation(master_id=master.database_id, slave_id=slave.database_id) session.add(relation) session.flush() return self._bond_database(session, master, slave, relation, **kwargs)
def delete_database(self, database_id, master, **kwargs): """delete master database intance""" session = endpoint_session() query = model_query(session, GopDatabase, filter=GopDatabase.database_id == database_id) if master: query = query.options( joinedload(GopDatabase.schemas, innerjoin=False)) else: query = query.options( joinedload(GopDatabase.quotes, innerjoin=False)) with session.begin(): _database = query.one() _result = dict(database_id=_database.database_id, slave=_database.slave, impl=_database.impl, dbtype=_database.dbtype, dbversion=_database.dbversion) # 删除主库 if master: if _database.slave: raise exceptions.AcceptableDbError( 'Target database is a salve database') if _database.schemas or _database.slaves: raise exceptions.AcceptableDbError( 'can not delete master database, slaves or schemas exist' ) if model_count_with_key(session, SchemaQuote.qdatabase_id, filter=SchemaQuote.qdatabase_id == _database.database_id): raise exceptions.AcceptableDbError( 'Database in schema quote list') with self._delete_database(session, _database, **kwargs) as address: host = address[0] port = address[1] _result.setdefault('host', host) _result.setdefault('port', port) query.delete() # 删除从库 else: if not _database.slave: raise exceptions.AcceptableDbError( 'Target database is not a slave database') if _database.quotes: raise exceptions.AcceptableDbError( 'Target slave database in schema quote list') _masters = [ m[0] for m in model_query(session, GopSalveRelation.master_id, filter=GopSalveRelation.slave_id == database_id).all() ] if _masters: masters = model_query( session, GopDatabase, filter=and_(GopDatabase.database_id.in_(_masters), GopDatabase.slave == 0)) if len(_masters) != len(masters): raise exceptions.UnAcceptableDbError( 'Target slave database master missed') raise exceptions.AcceptableDbError( 'Slave is bond to masters, unbond before delete') with self._delete_database(session, _database, **kwargs) as address: query.delete() host = address[0] port = address[1] _result.setdefault('host', host) _result.setdefault('port', port) return _result
with session.begin(): query = model_query(session, AsyncRequest, filter={'request_id': request_row.request_id}) rets = query.all() print rets ret = query.first() print ret, ret.to_dict() print 'scalar:', print query.scalar() print '~~~~~~~~~~~~~~~~~~~~~~~~~~' print '~~~~~test function~~~~~~~' print 'count agent', model_count_with_key(session, Agent) print 'count agent.agent_id', model_count_with_key(session, Agent.agent_id) print 'max agent.agent_id', model_max_with_key(session, Agent.agent_id) print 'max + 1 agent.agent_id', model_autoincrement_id(session, Agent.agent_id) print '~~~~~test function finish~~~~~~~' print '~~~~~test CASCADE~~~~~~~' print '~~~~~~~~~~CASCADE of endpoint~~~~~~~~~~~~~~~~' endpont = AgentEndpoint() endpont.endpoint = 'test' agent_row.endpoints = [ endpont, ] with session.begin(): session.add(agent_row)
def merge(self, req, body=None): """合服接口,用于合服, 部分代码和create代码一直,未整合""" body = body or {} jsonutils.schema_validate(body, self.MERGEAPPENTITYS) group_id = body.pop('group_id') # 需要合并的实体 entitys = list(set(body.pop('entitys'))) entitys.sort() session = endpoint_session() # 安装文件信息 appfile = body.pop(common.APPFILE) # 选择合并后实例运行服务器 agent_id = body.get('agent_id') or self._agentselect( req, common.GAMESERVER, **body) # 选择合并后实体数据库 databases = self._dbselect(req, common.GAMESERVER, **body) opentime = body.get('opentime') # 合服任务ID uuid = uuidutils.generate_uuid() # chiefs信息初始化 query = model_query(session, AppEntity, filter=and_( AppEntity.group_id == group_id, AppEntity.objtype.in_( [common.GMSERVER, common.CROSSSERVER]))) # 找到同组的gm和战场服 gm = None cross = None crosss = [] # 默认平台识标 platform = None # 锁组 glock = get_gamelock() with glock.grouplock(group_id): if model_count_with_key(session, MergeEntity, filter=MergeEntity.entity.in_(entitys)): raise InvalidArgument('Target entity merged or in mergeing') for appentity in query: if appentity.status != common.OK: continue if appentity.objtype == common.GMSERVER: gm = appentity else: crosss.append(appentity) if not gm: raise InvalidArgument( 'Group not exist or gm not active/exist?') if not crosss: raise InvalidArgument('Group has no cross server?') if not body.get('cross_id'): cross = crosss[0] else: for appentity in crosss: if appentity.entity == body.get('cross_id'): cross = appentity break if not cross: raise InvalidArgument('cross server can not be found?') # 获取实体相关服务器信息(端口/ip) maps = entity_controller.shows(endpoint=common.NAME, entitys=[gm.entity, cross.entity]) chiefs = dict() # 战场与GM服务器信息 for chief in (cross, gm): chiefmetadata = maps.get(chief.entity).get('metadata') ports = maps.get(chief.entity).get('ports') if not chiefmetadata: raise InvalidArgument('%s.%d is offline' % (chief.objtype, chief.entity)) need = common.POSTS_COUNT[chief.objtype] if need and len(ports) != need: raise InvalidArgument( '%s.%d port count error, ' 'find %d, need %d' % (chief.objtype, chief.entity, len(ports), need)) chiefs.setdefault( chief.objtype, dict(entity=chief.entity, ports=ports, local_ip=chiefmetadata.get('local_ip'))) # 需要合服的实体 appentitys = [] query = model_query(session, AppEntity, filter=and_(AppEntity.group_id == group_id, AppEntity.entity.in_(entitys))) query = query.options(joinedload(AppEntity.areas, innerjoin=False)) with session.begin(): for appentity in query: if appentity.objtype != common.GAMESERVER: raise InvalidArgument( 'Target entity %d is not %s' % (appentity.entity, common.GAMESERVER)) if appentity.status != common.UNACTIVE: raise InvalidArgument( 'Target entity %d is not unactive' % appentity.entity) if not appentity.areas: raise InvalidArgument('Target entity %d has no area?' % appentity.entity) if appentity.versions: raise InvalidArgument( 'Traget entity %d version is not None' % appentity.entity) if platform is None: platform = appentity.platform else: # 区服平台不相同, 位操作合并platform platform = platform | appentity.platform appentitys.append(appentity) if not opentime: opentime = appentity.opentime if len(appentitys) != len(entitys): raise InvalidArgument('Can not match entitys count') # 完整的rpc数据包,准备发送合服命令到agent body = dict(appfile=appfile, databases=databases, opentime=opentime, chiefs=chiefs, uuid=uuid, entitys=entitys) body.setdefault('finishtime', rpcfinishtime()[0] + 5) try: create_result = entity_controller.create( req=req, agent_id=agent_id, endpoint=common.NAME, body=body, action='merge')['data'][0] except RpcResultError as e: LOG.error('Create entity rpc call fail: %s' % e.message) raise InvalidArgument(e.message) mergetd_entity = create_result.get('entity') rpc_result = create_result.get('notify') LOG.info('Merge to entity %d, agent %d' % (mergetd_entity, agent_id)) LOG.debug('Entity controller merge rpc result %s' % str(rpc_result)) # 插入实体信息 appentity = AppEntity(entity=mergetd_entity, agent_id=agent_id, group_id=group_id, objtype=common.GAMESERVER, cross_id=cross.entity, opentime=opentime, platform=platform) session.add(appentity) session.flush() # 插入数据库绑定信息 if rpc_result.get('databases'): self._bondto(session, mergetd_entity, rpc_result.get('databases')) else: LOG.error('New entity database miss') # 插入合服记录 mtask = MergeTask(uuid=uuid, entity=mergetd_entity, mergetime=int(time.time())) session.add(mtask) session.flush() for _appentity in appentitys: session.add( MergeEntity(entity=_appentity.entity, uuid=uuid)) session.flush() # 批量修改被合并服的状态 query.update({'status': common.MERGEING}, synchronize_session=False) session.flush() port_controller.unsafe_create(agent_id, common.NAME, mergetd_entity, rpc_result.get('ports')) # agent 后续通知 threadpool.add_thread(entity_controller.post_create_entity, appentity.entity, common.NAME, objtype=common.GAMESERVER, status=common.UNACTIVE, opentime=opentime, group_id=group_id, areas=[]) # 添加端口 # threadpool.add_thread(port_controller.unsafe_create, # agent_id, common.NAME, mergetd_entity, rpc_result.get('ports')) return resultutils.results( result='entitys is mergeing', data=[dict(uuid=uuid, entitys=entitys, entity=mergetd_entity)])