def remove_host(self, host_id): if host_id not in self.hosts: logger.warning( "Attempted to remove host that does not exists", "server", server_id=self.id, host_id=host_id ) return logger.debug("Removing host from server", "server", server_id=self.id, host_id=host_id) self.hosts.remove(host_id) response = self.collection.update( {"_id": self.id, "instances.host_id": host_id}, {"$pull": {"hosts": host_id, "instances": {"host_id": host_id}}, "$inc": {"instances_count": -1}}, ) if response["updatedExisting"]: self.publish("start", extra={"prefered_hosts": host.get_prefered_hosts(self.hosts, self.replica_count)}) doc = self.collection.find_and_modify( {"_id": self.id}, {"$pull": {"hosts": host_id}}, {"hosts": True}, new=True ) if doc and not doc["hosts"]: self.status = OFFLINE self.commit("status")
def remove_host(self, host_id): if host_id not in self.hosts: logger.warning( 'Attempted to remove host that does not exists', 'server', server_id=self.id, host_id=host_id, ) return logger.debug( 'Removing host from server', 'server', server_id=self.id, host_id=host_id, ) self.hosts.remove(host_id) response = self.collection.update( { '_id': self.id, 'instances.host_id': host_id, }, { '$pull': { 'hosts': host_id, 'instances': { 'host_id': host_id, }, }, '$inc': { 'instances_count': -1, }, }) if response['updatedExisting']: self.publish('start', extra={ 'prefered_hosts': host.get_prefered_hosts(self.hosts, self.replica_count), }) doc = self.collection.find_and_modify({ '_id': self.id, }, { '$pull': { 'hosts': host_id, }, }, { 'hosts': True, }, new=True) if doc and not doc['hosts']: self.status = OFFLINE self.commit('status')
def remove_host(self, host_id): if host_id not in self.hosts: logger.warning('Attempted to remove host that does not exists', 'server', server_id=self.id, host_id=host_id, ) return logger.debug('Removing host from server', 'server', server_id=self.id, host_id=host_id, ) self.hosts.remove(host_id) response = self.collection.update({ '_id': self.id, 'instances.host_id': host_id, }, { '$pull': { 'hosts': host_id, 'instances': { 'host_id': host_id, }, }, '$inc': { 'instances_count': -1, }, }) if response['updatedExisting']: self.publish('start', extra={ 'prefered_hosts': host.get_prefered_hosts( self.hosts, self.replica_count), }) doc = self.collection.find_and_modify({ '_id': self.id, }, { '$pull': { 'hosts': host_id, }, }, { 'hosts': True, }, new=True) if doc and not doc['hosts']: self.status = OFFLINE self.commit('status')
def start(self, timeout=None): timeout = timeout or settings.vpn.op_timeout cursor_id = self.get_cursor_id() if self.status != OFFLINE: return if not self.dh_params: self.generate_dh_param() return if not self.organizations: raise ServerMissingOrg('Server cannot be started ' + \ 'without any organizations', { 'server_id': self.id, }) self.pre_start_check() start_timestamp = utils.now() response = self.collection.update( { '_id': self.id, 'status': OFFLINE, 'instances_count': 0, }, { '$set': { 'status': ONLINE, 'pool_cursor': None, 'start_timestamp': start_timestamp, 'availability_group': self.get_best_availability_group(), } }) if not response['updatedExisting']: raise ServerInstanceSet('Server instances already running. %r', { 'server_id': self.id, }) self.clients_pool_collection.remove({ 'server_id': self.id, }) self.status = ONLINE self.start_timestamp = start_timestamp replica_count = min(self.replica_count, len(self.hosts)) started_count = 0 error_count = 0 try: self.publish('start', extra={ 'prefered_hosts': host.get_prefered_hosts(self.hosts, replica_count), }) for x_timeout in (4, timeout): for msg in self.subscribe(cursor_id=cursor_id, timeout=x_timeout): message = msg['message'] if message == 'started': started_count += 1 if started_count + error_count >= replica_count: break elif message == 'error': error_count += 1 if started_count + error_count >= replica_count: break if started_count: break if not started_count: if error_count: raise ServerStartError('Server failed to start', { 'server_id': self.id, }) else: raise ServerStartError('Server start timed out', { 'server_id': self.id, }) except: self.publish('force_stop') self.collection.update({ '_id': self.id, }, { '$set': { 'status': OFFLINE, 'instances': [], 'instances_count': 0, } }) self.status = OFFLINE self.instances = [] self.instances_count = 0 raise
def start(self, timeout=None): timeout = timeout or settings.vpn.op_timeout cursor_id = self.get_cursor_id() if self.status != OFFLINE: return if not self.dh_params: self.generate_dh_param() return if not self.organizations: raise ServerMissingOrg('Server cannot be started ' + \ 'without any organizations', { 'server_id': self.id, }) self.pre_start_check() start_timestamp = utils.now() response = self.collection.update({ '_id': self.id, 'status': OFFLINE, 'instances_count': 0, }, {'$set': { 'status': ONLINE, 'start_timestamp': start_timestamp, }}) if not response['updatedExisting']: raise ServerInstanceSet('Server instances already running. %r', { 'server_id': self.id, }) self.status = ONLINE self.start_timestamp = start_timestamp replica_count = min(self.replica_count, len(self.hosts)) started_count = 0 error_count = 0 try: self.publish('start', extra={ 'prefered_hosts': host.get_prefered_hosts( self.hosts, replica_count), }) for x_timeout in (4, timeout): for msg in self.subscribe(cursor_id=cursor_id, timeout=x_timeout): message = msg['message'] if message == 'started': started_count += 1 if started_count + error_count >= replica_count: break elif message == 'error': error_count += 1 if started_count + error_count >= replica_count: break if started_count: break if not started_count: if error_count: raise ServerStartError('Server failed to start', { 'server_id': self.id, }) else: raise ServerStartError('Server start timed out', { 'server_id': self.id, }) except: self.publish('force_stop') self.collection.update({ '_id': self.id, }, {'$set': { 'status': OFFLINE, 'instances': [], 'instances_count': 0, }}) self.status = OFFLINE self.instances = [] self.instances_count = 0 raise
def task(self): try: timestamp_spec = utils.now() - datetime.timedelta( seconds=settings.vpn.server_ping_ttl) docs = self.server_collection.find({ 'instances.ping_timestamp': {'$lt': timestamp_spec}, }, { '_id': True, 'instances': True, }) yield for doc in docs: for instance in doc['instances']: if instance['ping_timestamp'] < timestamp_spec: self.server_collection.update({ '_id': doc['_id'], 'instances.instance_id': instance['instance_id'], }, { '$pull': { 'instances': { 'instance_id': instance['instance_id'], }, }, '$inc': { 'instances_count': -1, }, }) yield docs = self.host_collection.find({ 'status': ONLINE, }, { '_id': True, 'availability_group': True, }) yield hosts_group = {} for doc in docs: hosts_group[doc['_id']] = doc.get( 'availability_group', DEFAULT) yield response = self.server_collection.aggregate([ {'$match': { 'status': ONLINE, 'start_timestamp': {'$lt': timestamp_spec}, }}, {'$project': { '_id': True, 'hosts': True, 'instances': True, 'replica_count': True, 'availability_group': True, 'offline_instances_count': { '$subtract': [ '$replica_count', '$instances_count', ], } }}, {'$match': { 'offline_instances_count': {'$gt': 0}, }}, ]) yield for doc in response: cur_avail_group = doc.get('availability_group', DEFAULT) hosts_set = set(doc['hosts']) group_best = None group_len_max = 0 server_groups = collections.defaultdict(set) for hst in hosts_set: avail_zone = hosts_group.get(hst) if not avail_zone: continue server_groups[avail_zone].add(hst) group_len = len(server_groups[avail_zone]) if group_len > group_len_max: group_len_max = group_len group_best = avail_zone elif group_len == group_len_max and \ avail_zone == cur_avail_group: group_best = avail_zone if cur_avail_group != group_best: logger.info( 'Rebalancing server availability group', 'server', server_id=doc['_id'], current_availability_group=cur_avail_group, new_availability_group=group_best, ) self.server_collection.update({ '_id': doc['_id'], 'status': ONLINE, }, {'$set': { 'instances': [], 'instances_count': 0, 'availability_group': group_best, }}) messenger.publish('servers', 'rebalance', extra={ 'server_id': doc['_id'], 'availability_group': group_best, }) prefered_hosts = server_groups[group_best] else: prefered_hosts = server_groups[cur_avail_group] active_hosts = set( [x['host_id'] for x in doc['instances']]) prefered_hosts = list(prefered_hosts - active_hosts) if not prefered_hosts: continue logger.info('Recovering server state', 'server', server_id=doc['_id'], prefered_hosts=prefered_hosts, ) messenger.publish('servers', 'start', extra={ 'server_id': doc['_id'], 'send_events': True, 'prefered_hosts': host.get_prefered_hosts( prefered_hosts, doc['replica_count']) }) except GeneratorExit: raise except: logger.exception('Error checking server states', 'tasks')
def thread(): platforms = list(DESKTOP_PLATFORMS) start_timestamp = datetime.datetime(2015, 12, 28, 4, 1, 0) hosts_collection = mongo.get_collection('hosts') servers_collection = mongo.get_collection('servers') clients_collection = mongo.get_collection('clients') clients_collection.remove({}) for hst in host.iter_hosts(): hosts_collection.update({ '_id': hst.id, }, {'$set': { 'server_count': 0, 'device_count': 0, 'cpu_usage': 0, 'mem_usage': 0, 'thread_count': 0, 'open_file_count': 0, 'status': ONLINE, 'start_timestamp': start_timestamp, 'ping_timestamp': start_timestamp, 'auto_public_address': None, 'auto_public_address6': None, 'auto_public_host': hst.name + '.pritunl.com', 'auto_public_host6': hst.name + '.pritunl.com', }}) for svr in server.iter_servers(): prefered_hosts = host.get_prefered_hosts( svr.hosts, svr.replica_count) instances = [] for hst in prefered_hosts: instances.append({ 'instance_id': utils.ObjectId(), 'host_id': hst, 'ping_timestamp': utils.now(), }) servers_collection.update({ '_id': svr.id, }, {'$set': { 'status': ONLINE, 'pool_cursor': None, 'start_timestamp': start_timestamp, 'availability_group': DEFAULT, 'instances': instances, 'instances_count': len(instances), }}) for org in svr.iter_orgs(): for usr in org.iter_users(): if usr.type != CERT_CLIENT: continue virt_address = svr.get_ip_addr(org.id, usr.id) virt_address6 = svr.ip4to6(virt_address) doc = { '_id': utils.ObjectId(), 'user_id': usr.id, 'server_id': svr.id, 'host_id': settings.local.host_id, 'timestamp': start_timestamp, 'platform': random.choice(platforms), 'type': CERT_CLIENT, 'device_name': utils.random_name(), 'mac_addr': utils.rand_str(16), 'network': svr.network, 'real_address': str( ipaddress.IPAddress(100000000 + random.randint( 0, 1000000000))), 'virt_address': virt_address, 'virt_address6': virt_address6, 'host_address': settings.local.host.local_addr, 'host_address6': settings.local.host.local_addr6, 'dns_servers': [], 'dns_suffix': None, 'connected_since': int(start_timestamp.strftime('%s')), } clients_collection.insert(doc) for lnk in link.iter_links(): lnk.status = ONLINE lnk.commit() for location in lnk.iter_locations(): active = False for hst in location.iter_hosts(): if not active: hst.active = True active = True hst.status = AVAILABLE hst.commit(('active', 'status')) logger.info('Demo initiated', 'demo')
def task(self): if settings.app.demo_mode: return try: timestamp = utils.now() timestamp_spec = timestamp - datetime.timedelta( seconds=settings.vpn.server_ping_ttl) docs = self.server_collection.find( { 'instances.ping_timestamp': { '$lt': timestamp_spec }, }, { '_id': True, 'instances': True, }) yield for doc in docs: for instance in doc['instances']: if instance['ping_timestamp'] < timestamp_spec: logger.warning( 'Removing instance doc', 'server', server_id=doc['_id'], instance_id=instance['instance_id'], cur_timestamp=timestamp, ttl_timestamp=timestamp_spec, ping_timestamp=instance['ping_timestamp'], ) self.server_collection.update( { '_id': doc['_id'], 'instances.instance_id': instance['instance_id'], }, { '$pull': { 'instances': { 'instance_id': instance['instance_id'], }, }, '$inc': { 'instances_count': -1, }, }) yield docs = self.host_collection.find({ 'status': ONLINE, }, { '_id': True, 'availability_group': True, }) yield hosts_group = {} for doc in docs: hosts_group[doc['_id']] = doc.get('availability_group', DEFAULT) yield response = self.server_collection.aggregate([ { '$match': { 'status': ONLINE, 'start_timestamp': { '$lt': timestamp_spec }, } }, { '$project': { '_id': True, 'hosts': True, 'instances': True, 'replica_count': True, 'availability_group': True, 'offline_instances_count': { '$subtract': [ '$replica_count', '$instances_count', ], } } }, { '$match': { 'offline_instances_count': { '$gt': 0 }, } }, ]) yield recover_count = 0 for doc in response: cur_avail_group = doc.get('availability_group', DEFAULT) hosts_set = set(doc['hosts']) group_best = None group_len_max = 0 server_groups = collections.defaultdict(set) for hst in hosts_set: avail_zone = hosts_group.get(hst) if not avail_zone: continue server_groups[avail_zone].add(hst) group_len = len(server_groups[avail_zone]) if group_len > group_len_max: group_len_max = group_len group_best = avail_zone elif group_len == group_len_max and \ avail_zone == cur_avail_group: group_best = avail_zone if group_best and cur_avail_group != group_best: logger.info( 'Rebalancing server availability group', 'server', server_id=doc['_id'], current_availability_group=cur_avail_group, new_availability_group=group_best, ) self.server_collection.update( { '_id': doc['_id'], 'status': ONLINE, }, { '$set': { 'instances': [], 'instances_count': 0, 'availability_group': group_best, } }) messenger.publish('servers', 'rebalance', extra={ 'server_id': doc['_id'], 'availability_group': group_best, }) prefered_hosts = server_groups[group_best] else: prefered_hosts = server_groups[cur_avail_group] active_hosts = set([x['host_id'] for x in doc['instances']]) prefered_hosts = list(prefered_hosts - active_hosts) if not prefered_hosts: continue if recover_count >= 3: continue recover_count += 1 logger.info( 'Recovering server state', 'server', server_id=doc['_id'], prefered_hosts=prefered_hosts, ) messenger.publish('servers', 'start', extra={ 'server_id': doc['_id'], 'send_events': True, 'prefered_hosts': host.get_prefered_hosts( prefered_hosts, doc['replica_count']) }) except GeneratorExit: raise except: logger.exception('Error checking server states', 'tasks')
def task(self): try: timestamp_spec = utils.now() - datetime.timedelta( seconds=settings.vpn.server_ping_ttl) docs = self.server_collection.find({ 'instances.ping_timestamp': {'$lt': timestamp_spec}, }, { '_id': True, 'instances': True, }) yield for doc in docs: for instance in doc['instances']: if instance['ping_timestamp'] < timestamp_spec: self.server_collection.update({ '_id': doc['_id'], 'instances.instance_id': instance['instance_id'], }, { '$pull': { 'instances': { 'instance_id': instance['instance_id'], }, }, '$inc': { 'instances_count': -1, }, }) yield response = self.server_collection.aggregate([ {'$match': { 'status': ONLINE, 'start_timestamp': {'$lt': timestamp_spec}, }}, {'$project': { '_id': True, 'hosts': True, 'instances': True, 'replica_count': True, 'offline_instances_count': { '$subtract': [ '$replica_count', '$instances_count', ], } }}, {'$match': { 'offline_instances_count': {'$gt': 0}, }}, ]) yield for doc in response: active_hosts = set([x['host_id'] for x in doc['instances']]) hosts = list(set(doc['hosts']) - active_hosts) if not hosts: continue messenger.publish('servers', 'start', extra={ 'server_id': doc['_id'], 'send_events': True, 'prefered_hosts': host.get_prefered_hosts( hosts, doc['replica_count']) }) except GeneratorExit: raise except: logger.exception('Error checking server states', 'tasks')
def thread(): platforms = list(DESKTOP_PLATFORMS) start_timestamp = datetime.datetime(2015, 12, 28, 4, 1, 0) hosts_collection = mongo.get_collection('hosts') servers_collection = mongo.get_collection('servers') clients_collection = mongo.get_collection('clients') clients_collection.remove({}) for hst in host.iter_hosts(): hosts_collection.update({ '_id': hst.id, }, { '$set': { 'server_count': 0, 'device_count': 0, 'cpu_usage': 0, 'mem_usage': 0, 'thread_count': 0, 'open_file_count': 0, 'status': ONLINE, 'start_timestamp': start_timestamp, 'ping_timestamp': start_timestamp, 'auto_public_address': None, 'auto_public_address6': None, 'auto_public_host': hst.name + '.pritunl.com', 'auto_public_host6': hst.name + '.pritunl.com', } }) for svr in server.iter_servers(): prefered_hosts = host.get_prefered_hosts(svr.hosts, svr.replica_count) instances = [] for hst in prefered_hosts: instances.append({ 'instance_id': utils.ObjectId(), 'host_id': hst, 'ping_timestamp': utils.now(), }) servers_collection.update({ '_id': svr.id, }, { '$set': { 'status': ONLINE, 'pool_cursor': None, 'start_timestamp': start_timestamp, 'availability_group': DEFAULT, 'instances': instances, 'instances_count': len(instances), } }) for org in svr.iter_orgs(): for usr in org.iter_users(): if usr.type != CERT_CLIENT: continue virt_address = svr.get_ip_addr(org.id, usr.id) virt_address6 = svr.ip4to6(virt_address) doc = { '_id': utils.ObjectId(), 'user_id': usr.id, 'server_id': svr.id, 'host_id': settings.local.host_id, 'timestamp': start_timestamp, 'platform': random.choice(platforms), 'type': CERT_CLIENT, 'device_name': utils.random_name(), 'mac_addr': utils.rand_str(16), 'network': svr.network, 'real_address': str( ipaddress.IPAddress( 100000000 + random.randint(0, 1000000000))), 'virt_address': virt_address, 'virt_address6': virt_address6, 'host_address': settings.local.host.local_addr, 'host_address6': settings.local.host.local_addr6, 'dns_servers': [], 'dns_suffix': None, 'connected_since': int(start_timestamp.strftime('%s')), } clients_collection.insert(doc) for lnk in link.iter_links(): lnk.status = ONLINE lnk.commit() for location in lnk.iter_locations(): active = False for hst in location.iter_hosts(): if not active: hst.active = True active = True hst.status = AVAILABLE hst.commit(('active', 'status')) logger.info('Demo initiated', 'demo')
def task(self): try: timestamp_spec = utils.now() - datetime.timedelta( seconds=settings.vpn.server_ping_ttl) docs = self.server_collection.find({ 'instances.ping_timestamp': {'$lt': timestamp_spec}, }, { '_id': True, 'instances': True, }) yield for doc in docs: for instance in doc['instances']: if instance['ping_timestamp'] < timestamp_spec: self.server_collection.update({ '_id': doc['_id'], 'instances.instance_id': instance['instance_id'], }, { '$pull': { 'instances': { 'instance_id': instance['instance_id'], }, }, '$inc': { 'instances_count': -1, }, }) yield response = self.server_collection.aggregate([ {'$match': { 'status': ONLINE, 'start_timestamp': {'$lt': timestamp_spec}, }}, {'$project': { '_id': True, 'hosts': True, 'instances': True, 'replica_count': True, 'offline_instances_count': { '$subtract': [ '$replica_count', '$instances_count', ], } }}, {'$match': { 'offline_instances_count': {'$gt': 0}, }}, ]) yield for doc in response: active_hosts = set([x['host_id'] for x in doc['instances']]) hosts = list(set(doc['hosts']) - active_hosts) if not hosts: continue messenger.publish('servers', 'start', extra={ 'server_id': doc['_id'], 'send_events': True, 'prefered_hosts': host.get_prefered_hosts( hosts, doc['replica_count']) }) except GeneratorExit: raise except: logger.exception('Error checking server states', 'tasks') yield interrupter_sleep(settings.vpn.server_ping)
def start(self, timeout=None): timeout = timeout or settings.vpn.op_timeout cursor_id = self.get_cursor_id() if self.status != OFFLINE: return if not self.dh_params: self.generate_dh_param() return if not self.organizations: raise ServerMissingOrg("Server cannot be started " + "without any organizations", {"server_id": self.id}) self.pre_start_check() start_timestamp = utils.now() response = self.collection.update( {"_id": self.id, "status": OFFLINE, "instances_count": 0}, {"$set": {"status": ONLINE, "start_timestamp": start_timestamp}}, ) if not response["updatedExisting"]: raise ServerInstanceSet("Server instances already running. %r", {"server_id": self.id}) self.status = ONLINE self.start_timestamp = start_timestamp replica_count = min(self.replica_count, len(self.hosts)) started_count = 0 error_count = 0 try: self.publish("start", extra={"prefered_hosts": host.get_prefered_hosts(self.hosts, replica_count)}) for x_timeout in (4, timeout): for msg in self.subscribe(cursor_id=cursor_id, timeout=x_timeout): message = msg["message"] if message == "started": started_count += 1 if started_count + error_count >= replica_count: break elif message == "error": error_count += 1 if started_count + error_count >= replica_count: break if started_count: break if not started_count: if error_count: raise ServerStartError("Server failed to start", {"server_id": self.id}) else: raise ServerStartError("Server start timed out", {"server_id": self.id}) except: self.publish("force_stop") self.collection.update( {"_id": self.id}, {"$set": {"status": OFFLINE, "instances": [], "instances_count": 0}} ) self.status = OFFLINE self.instances = [] self.instances_count = 0 raise