def _expire(): try: BaseContorller._agent_metadata_expire(agent_id, expire + fix) except AgentMetadataMiss: # 元数据丢失, 通知agent重新上报 rpc = get_client() # send to rpc server rpc.cast(targetutils.target_all(fanout=True), ctxt={'agents': [agent_id, ]}, msg={'method': 'flush_metadata', 'args': {'expire': expire}})
def show(self, req, agent_id, body=None): body = body or {} show_ports = body.get('ports', False) show_entitys = body.get('entitys', False) session = get_session(readonly=True) joins = joinedload(Agent.endpoints, innerjoin=False) if show_entitys: joins = joins.joinedload(AgentEndpoint.entitys, innerjoin=False) if show_ports: joins = joins.joinedload(AgentEntity.ports, innerjoin=False) query = model_query(session, Agent).options(joins) agent = query.filter_by(agent_id=agent_id).one() result = resultutils.results(total=1, pagenum=0, result='Show agent success') endpoints = {} for endpoint in agent.endpoints: endpoints[endpoint.endpoint] = [] if show_entitys: for entity in endpoint.entitys: _entity = {'entity': entity.entity, 'ports': []} endpoints[endpoint.endpoint].append(_entity) if show_ports: for port in entity.ports: _entity['ports'].append(port['port']) result_data = dict(agent_id=agent.agent_id, host=agent.host, agent_type=agent.agent_type, status=agent.status, cpu=agent.cpu, memory=agent.memory, disk=agent.disk, ports_range=jsonutils.safe_loads_as_bytes(agent.ports_range) or [], endpoints=endpoints, metadata=BaseContorller.agent_metadata(agent_id), ) result['data'].append(result_data) return result
def show(self, req, endpoint, entity, body=None): body = body or {} show_ports = body.get('ports', False) endpoint = validateutils.validate_endpoint(endpoint) entity = int(entity) session = get_session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) if show_ports: query = query.options( joinedload(AgentEntity.ports, innerjoin=False)) _entity = query.one_or_none() if not _entity: raise InvalidArgument('no entity found for %s' % endpoint) # return resultutils.results(result='no entity found', resultcode=manager_common.RESULT_ERROR) return resultutils.results( result='show entity success', data=[ dict(endpoint=_entity.endpoint, agent_id=_entity.agent_id, metadata=BaseContorller.agent_metadata(_entity.agent_id), entity=_entity.entity, ports=sorted([x.port for x in _entity.ports]) if show_ports else []) ])
def online(self, req, host, body): """call buy agent when a agent start, it will cache agent ipaddr """ try: host = validators['type:hostname'](host) agent_type = body.pop('agent_type') metadata = body.pop('metadata') expire = body.pop('expire') except KeyError as e: raise InvalidArgument('Can not find argument: %s' % e.message) except ValueError as e: raise InvalidArgument('Argument value type error: %s' % e.message) except InvalidInput as e: raise InvalidArgument(e.message) session = get_session(readonly=True) query = model_query(session, Agent, filter=(and_(Agent.status > manager_common.DELETED, Agent.agent_type == agent_type, Agent.host == host))) agent = query.one_or_none() if not agent: LOG.info('Cache online called but no Agent found') ret = {'agent_id': None} else: self.agent_id_check(agent.agent_id) local_ip = metadata.get('local_ip') external_ips = str(metadata.get('external_ips')) LOG.debug( 'Cache online called. agent_id:%(agent_id)s, type:%(agent_type)s, ' 'host:%(host)s, local_ip:%(local_ip)s, external_ips:%(external_ips)s' % { 'agent_id': agent.agent_id, 'agent_type': agent_type, 'host': host, 'local_ip': local_ip, 'external_ips': external_ips }) ret = {'agent_id': agent.agent_id} BaseContorller._agent_metadata_flush(agent.agent_id, metadata, expire=expire) result = resultutils.results( result='Cache online function run success') result['data'].append(ret) return result
def delete(self, req, agent_id, body=None): """call buy agent""" # if force is true # will not notify agent, just delete agent from database body = body or {} force = body.get('force', False) rpc = get_client() global_data = get_global() metadata = None with global_data.delete_agent(agent_id) as agent: if not force: metadata = BaseContorller.agent_metadata(agent.agent_id) if metadata is None: raise RpcPrepareError('Can not delete offline agent, try force') agent_ipaddr = metadata.get('local_ip') secret = uuidutils.generate_uuid() # tell agent wait delete finishtime, timeout = rpcfinishtime() delete_agent_precommit = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': finishtime}, msg={'method': 'delete_agent_precommit', 'args': {'agent_id': agent.agent_id, 'agent_type': agent.agent_type, 'host': agent.host, 'agent_ipaddr': agent_ipaddr, 'secret': secret} }, timeout=timeout) if not delete_agent_precommit: raise RpcResultError('delete_agent_precommit result is None') if delete_agent_precommit.get('resultcode') != manager_common.RESULT_SUCCESS: return resultutils.results(result=delete_agent_precommit.get('result'), resultcode=manager_common.RESULT_ERROR) # if not force: # tell agent delete itself finishtime = rpcfinishtime()[0] LOG.info('Delete agent %s postcommit with secret %s' % (agent_ipaddr, secret)) rpc.cast(targetutils.target_agent(agent), ctxt={'finishtime': finishtime}, msg={'method': 'delete_agent_postcommit', 'args': {'agent_id': agent.agent_id, 'agent_type': agent.agent_type, 'host': agent.host, 'agent_ipaddr': agent_ipaddr, 'secret': secret}}) def wapper(): rpc.cast(targetutils.target_rpcserver(fanout=True), msg={'method': 'deletesource', 'args': {'agent_id': agent_id}}) threadpool.add_thread(safe_func_wrapper, wapper, LOG) result = resultutils.results(result='Delete agent success', data=[dict(agent_id=agent.agent_id, host=agent.host, status=agent.status, metadata=metadata, ports_range=jsonutils.safe_loads_as_bytes(agent.ports_range) or []) ]) return result
def _check_file(agent_id, objtype, appfile): metadata = BaseContorller.agent_metadata(agent_id) if not metadata: return False target = targetutils.target_agent_by_string(metadata.get('agent_type'), metadata.get('host')) rpc = get_client() rpc_ret = rpc.call(target, ctxt={'agents': [agent_id, ]}, msg={'method': 'check_file', 'args': dict(objtype=objtype, appfile=appfile)}) if not rpc_ret: LOG.error('Rpc call result is None') return False if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: return False return True
def delete(self, req, endpoint, entity, body=None, action='delete'): body = body or {} notify = body.pop('notify', True) endpoint = validateutils.validate_endpoint(endpoint) entity = int(entity) session = get_session() glock = get_global().lock('entitys') result = 'delete entity success.' with glock(endpoint, [ entity, ]) as agents: with session.begin(): query = model_query(session, AgentEntity, filter=and_( AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) if notify: agent_id = agents.pop() metadata = BaseContorller.agent_metadata(agent_id) if not metadata: raise InvalidArgument('Agent not online or not exist') _entity = query.one_or_none() if not _entity: LOG.warning('Delete no entitys, but expect count 1') else: query.delete() if notify: target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = endpoint delete_result = self.notify_delete(target, agent_id, entity, body, action) if not delete_result: raise RpcResultError('delete entitys result is None') if delete_result.get( 'resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('delete entity fail %s' % delete_result.get('result')) result += delete_result.get('result') notify = delete_result return resultutils.results( result=result, data=[dict(entity=entity, endpoint=endpoint, notify=notify)])
def readlog(self, req, agent_id, body=None): """call by agent""" body = body or {} lines = body.get('lines', 10) rpc = get_client() metadata = BaseContorller.agent_metadata(agent_id) if metadata is None: raise RpcPrepareError('Can not get log from offline agent: %d' % agent_id) session = get_session() query = model_query(session, Agent, filter=Agent.agent_id == agent_id) agent = query.one() rpc_ret = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': rpcfinishtime()[0]}, msg={'method': 'readlog', 'args': {'lines': lines}}) if not rpc_ret: raise RpcResultError('Get log agent rpc result is None') if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('Get log agent rpc result: ' + rpc_ret.get('result')) return resultutils.results(result=rpc_ret.get('result'), data=[rpc_ret.get('uri')])
def readlog(self, req, endpoint, entity, body=None): body = body or {} path = body.get('path') lines = body.get('lines', 10) if not path or '..' in path: raise InvalidArgument('Path value error') endpoint = validateutils.validate_endpoint(endpoint) entity = int(entity) session = get_session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) _entity = query.one_or_none() if not _entity: raise InvalidArgument('no entity found for %s' % endpoint) metadata = BaseContorller.agent_metadata(_entity.agent_id) if not metadata: raise InvalidArgument('Can not get log from off line agent') target = targetutils.target_agent_by_string(manager_common.APPLICATION, metadata.get('host')) target.namespace = endpoint rpc = get_client() rpc_ret = rpc.call(target, ctxt={'finishtime': rpcfinishtime()[0]}, msg={ 'method': 'readlog', 'args': { 'entity': entity, 'path': path, 'lines': lines } }) if not rpc_ret: raise RpcResultError('Get %s.%d log rpc result is None' % (endpoint, entity)) if rpc_ret.get('resultcode') != manager_common.RESULT_SUCCESS: raise InvalidArgument(('Get %s.%d log agent rpc result: ' % (endpoint, entity)) + rpc_ret.get('result')) return resultutils.results(result=rpc_ret.get('result'), data=[rpc_ret.get('uri')])
def active(self, req, agent_id, body=None): """call buy client""" body = body or {} status = body.get('status', manager_common.ACTIVE) if status not in (manager_common.ACTIVE, manager_common.UNACTIVE): raise InvalidArgument('Argument status not right') rpc = get_client() session = get_session() query = model_query(session, Agent, filter=and_(Agent.agent_id == agent_id, Agent.status > manager_common.DELETED)) agent = query.one() # make sure agent is online metadata = BaseContorller.agent_metadata(agent.agent_id) if metadata is None: raise RpcPrepareError('Can not active or unactive a offline agent: %d' % agent_id) agent_ipaddr = metadata.get('local_ip') with session.begin(): agent.update({'status': status}) active_agent = rpc.call(targetutils.target_agent(agent), ctxt={'finishtime': rpcfinishtime()[0]}, msg={'method': 'active_agent', 'args': {'agent_id': agent_id, 'agent_ipaddr': agent_ipaddr, 'status': status} }) if not active_agent: raise RpcResultError('Active agent rpc result is None') if active_agent.pop('resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('Call agent active or unactive fail: ' + active_agent.get('result')) result = resultutils.results(result=active_agent.pop('result'), data=[dict(agent_id=agent.agent_id, host=agent.host, agent_type=agent.agent_type, metadata=metadata, status=agent.status) ]) return result
def shows(endpoint, entitys=None, agents=None, ports=True, metadata=True): entitys_map = {} endpoint = validateutils.validate_endpoint(endpoint) session = get_session(readonly=True) filters = [AgentEntity.endpoint == endpoint] if not entitys and not agents: return entitys_map if entitys: entitys = argutils.map_to_int(entitys) filters.append(AgentEntity.entity.in_(entitys)) if agents: agents = argutils.map_to_int(agents) filters.append(AgentEntity.agent_id.in_(agents)) if len(filters) > 1: filter = and_(*filters) else: filter = filters[0] query = model_query(session, AgentEntity, filter=filter) if ports: query = query.options( joinedload(AgentEntity.ports, innerjoin=False)) agents = set() for _entity in query: agents.add(_entity.agent_id) entitys_map[_entity.entity] = dict(agent_id=_entity.agent_id) if ports: entitys_map[_entity.entity].setdefault( 'ports', sorted([x.port for x in _entity.ports])) if metadata: agents_map = BaseContorller.agents_metadata(agents) for _entity in entitys_map: agent_id = entitys_map[_entity].get('agent_id') if metadata: entitys_map[_entity].setdefault('metadata', agents_map[agent_id]) return entitys_map
def post_create_entity(self, entity, endpoint, **kwargs): entity = int(entity) endpoint = validateutils.validate_endpoint(endpoint) session = get_session(readonly=True) query = model_query(session, AgentEntity, filter=and_(AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) _entity = query.one() metadata = BaseContorller.agent_metadata(_entity.agent_id) if not metadata: raise RpcPrepareError('Agent not online, can not sen post create') target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host'), ) target.namespace = endpoint body = dict(entity=entity) body.update(kwargs) rpc = get_client() rpc.cast(target, ctxt={ 'finishtime': body.pop('finishtime', rpcfinishtime()[0]), 'entitys': [ entity, ] }, msg={ 'method': 'post_create_entity', 'args': body }) return resultutils.results(result='notify post create success', data=[ dict(entity=entity, agent_id=_entity.agent_id, endpoint=endpoint) ])
def create(self, req, agent_id, endpoint, body=None, action='create'): body = body or {} endpoint = validateutils.validate_endpoint(endpoint) ports = body.pop('ports', None) notify = body.pop('notify', True) desc = body.pop('desc', None) session = get_session() metadata = None if ports: ports = argutils.map_with(ports, validators['type:port']) used_ports = model_count_with_key( session, AllocatedPort.port, filter=and_(AllocatedPort.agent_id == agent_id, AllocatedPort.port.in_(ports))) if used_ports: raise InvalidArgument('Ports has been used count %d' % used_ports) if notify: metadata = BaseContorller.agent_metadata(agent_id) # make sure agent is online if not metadata: raise RpcPrepareError( 'Can not create entity on a offline agent %d' % agent_id) entity = 0 glock = get_global().lock('agents') elock = get_global().lock('endpoint') result = 'add entity success.' with glock([ agent_id, ]): with elock(endpoint): with session.begin(subtransactions=True): query = model_query(session, Agent, filter=Agent.agent_id == agent_id) query = query.options( joinedload(Agent.endpoints, innerjoin=False)) agent = query.one() if agent.status != manager_common.ACTIVE: raise InvalidArgument( 'Create entity fail, agent status is not active') _endpoint = None for e in agent.endpoints: if endpoint == e.endpoint: _endpoint = e break if not _endpoint: raise InvalidArgument( 'Create entity fail, agent %d has no endpoint %s' % (agent_id, endpoint)) entity_autoincrement_id = model_autoincrement_id( session, AgentEntity.entity, filter=AgentEntity.endpoint == endpoint) entity = body.get('entity') or entity_autoincrement_id if entity < entity_autoincrement_id: raise InvalidArgument( 'Create entity fail, entity less then autoincrement id' ) _entity = AgentEntity(entity=entity, endpoint=endpoint, agent_id=agent_id, endpoint_id=_endpoint.id, desc=desc) session.add(_entity) session.flush() LOG.info('Create entity %s.%d with entity id %s' % (endpoint, entity, _entity.id)) if ports: for port in ports: session.add( AllocatedPort(port=port, agent_id=agent_id, endpoint_id=_endpoint.id, entity_id=entity.id, endpoint=endpoint, entity=entity)) session.flush() if notify: target = targetutils.target_agent(agent) target.namespace = endpoint create_result = self.notify_create( target, agent.agent_id, entity, body, action) if not create_result: raise RpcResultError( 'create entitys result is None') if create_result.get( 'resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('create entity fail %s' % create_result.get('result')) result += create_result.get('result') notify = create_result return resultutils.results(result=result, data=[ dict(entity=entity, agent_id=agent_id, metadata=metadata, endpoint=endpoint, ports=ports or [], notify=notify) ])
def migrate_with_out_data(endpoint, entity, new, body=None, drop_ports=True): session = get_session() glock = get_global().lock('entitys') with glock(endpoint, [ entity, ]) as agents: with session.begin(): _query = model_query(session, AgentEndpoint, filter=and_( AgentEndpoint.agent_id == new, AgentEndpoint.endpoint == endpoint)) _endpoint = _query.one_or_none() if not _endpoint: raise InvalidArgument( 'New agent not exist or has not endpoint %s' % endpoint) query = model_query(session, AgentEntity, filter=and_( AgentEntity.endpoint == endpoint, AgentEntity.entity == entity)) agent_id = agents.pop() metadata = BaseContorller.agent_metadata(agent_id) if not metadata: raise InvalidArgument('Agent not online or not exist') _agent_entity = query.one() if _agent_entity.agent_id != agent_id: raise CacheStoneError('Agent id not the same') if agent_id == new: raise InvalidArgument('Migrate agent is the same') ports_query = model_query( session, AllocatedPort, filter=and_(AllocatedPort.entity == entity, AllocatedPort.agent_id == agent_id)) if drop_ports: ports_query.delete() session.flush() else: for port in ports_query: port.agent_id = new port.endpoint_id = _endpoint.id try: session.flush() except DBDuplicateEntry: LOG.error('Migrate port fail, port duplicate') raise InvalidArgument('Port duplicate') _agent_entity.agent_id = new _agent_entity.endpoint_id = _endpoint.id session.flush() target = targetutils.target_agent_by_string( metadata.get('agent_type'), metadata.get('host')) target.namespace = endpoint LOG.debug( 'Migrate process call notify delete to source server') delete_result = EntityReuest.notify_delete( target, agent_id, entity, body) if not delete_result: raise RpcResultError('delete entitys result is None') if delete_result.get( 'resultcode') != manager_common.RESULT_SUCCESS: raise RpcResultError('delete entity fail %s' % delete_result.get('result')) LOG.info('Migrate process call notify delete success') yield