def remove_host(self, host_id):
if host_id not in self.hosts:
logger.warning(
"Attempted to remove host that does not exists", "server", server_id=self.id, host_id=host_id
)
return
logger.debug("Removing host from server", "server", server_id=self.id, host_id=host_id)
self.hosts.remove(host_id)
response = self.collection.update(
{"_id": self.id, "instances.host_id": host_id},
{"$pull": {"hosts": host_id, "instances": {"host_id": host_id}}, "$inc": {"instances_count": -1}},
)
if response["updatedExisting"]:
self.publish("start", extra={"prefered_hosts": host.get_prefered_hosts(self.hosts, self.replica_count)})
doc = self.collection.find_and_modify(
{"_id": self.id}, {"$pull": {"hosts": host_id}}, {"hosts": True}, new=True
)
if doc and not doc["hosts"]:
self.status = OFFLINE
self.commit("status")
def remove_host(self, host_id):
if host_id not in self.hosts:
logger.warning(
'Attempted to remove host that does not exists',
'server',
server_id=self.id,
host_id=host_id,
)
return
logger.debug(
'Removing host from server',
'server',
server_id=self.id,
host_id=host_id,
)
self.hosts.remove(host_id)
response = self.collection.update(
{
'_id': self.id,
'instances.host_id': host_id,
}, {
'$pull': {
'hosts': host_id,
'instances': {
'host_id': host_id,
},
},
'$inc': {
'instances_count': -1,
},
})
if response['updatedExisting']:
self.publish('start',
extra={
'prefered_hosts':
host.get_prefered_hosts(self.hosts,
self.replica_count),
})
doc = self.collection.find_and_modify({
'_id': self.id,
}, {
'$pull': {
'hosts': host_id,
},
}, {
'hosts': True,
},
new=True)
if doc and not doc['hosts']:
self.status = OFFLINE
self.commit('status')
def remove_host(self, host_id):
if host_id not in self.hosts:
logger.warning('Attempted to remove host that does not exists',
'server',
server_id=self.id,
host_id=host_id,
)
return
logger.debug('Removing host from server', 'server',
server_id=self.id,
host_id=host_id,
)
self.hosts.remove(host_id)
response = self.collection.update({
'_id': self.id,
'instances.host_id': host_id,
}, {
'$pull': {
'hosts': host_id,
'instances': {
'host_id': host_id,
},
},
'$inc': {
'instances_count': -1,
},
})
if response['updatedExisting']:
self.publish('start', extra={
'prefered_hosts': host.get_prefered_hosts(
self.hosts, self.replica_count),
})
doc = self.collection.find_and_modify({
'_id': self.id,
}, {
'$pull': {
'hosts': host_id,
},
}, {
'hosts': True,
}, new=True)
if doc and not doc['hosts']:
self.status = OFFLINE
self.commit('status')
def start(self, timeout=None):
timeout = timeout or settings.vpn.op_timeout
cursor_id = self.get_cursor_id()
if self.status != OFFLINE:
return
if not self.dh_params:
self.generate_dh_param()
return
if not self.organizations:
raise ServerMissingOrg('Server cannot be started ' + \
'without any organizations', {
'server_id': self.id,
})
self.pre_start_check()
start_timestamp = utils.now()
response = self.collection.update(
{
'_id': self.id,
'status': OFFLINE,
'instances_count': 0,
}, {
'$set': {
'status': ONLINE,
'pool_cursor': None,
'start_timestamp': start_timestamp,
'availability_group': self.get_best_availability_group(),
}
})
if not response['updatedExisting']:
raise ServerInstanceSet('Server instances already running. %r', {
'server_id': self.id,
})
self.clients_pool_collection.remove({
'server_id': self.id,
})
self.status = ONLINE
self.start_timestamp = start_timestamp
replica_count = min(self.replica_count, len(self.hosts))
started_count = 0
error_count = 0
try:
self.publish('start',
extra={
'prefered_hosts':
host.get_prefered_hosts(self.hosts,
replica_count),
})
for x_timeout in (4, timeout):
for msg in self.subscribe(cursor_id=cursor_id,
timeout=x_timeout):
message = msg['message']
if message == 'started':
started_count += 1
if started_count + error_count >= replica_count:
break
elif message == 'error':
error_count += 1
if started_count + error_count >= replica_count:
break
if started_count:
break
if not started_count:
if error_count:
raise ServerStartError('Server failed to start', {
'server_id': self.id,
})
else:
raise ServerStartError('Server start timed out', {
'server_id': self.id,
})
except:
self.publish('force_stop')
self.collection.update({
'_id': self.id,
}, {
'$set': {
'status': OFFLINE,
'instances': [],
'instances_count': 0,
}
})
self.status = OFFLINE
self.instances = []
self.instances_count = 0
raise
def start(self, timeout=None):
timeout = timeout or settings.vpn.op_timeout
cursor_id = self.get_cursor_id()
if self.status != OFFLINE:
return
if not self.dh_params:
self.generate_dh_param()
return
if not self.organizations:
raise ServerMissingOrg('Server cannot be started ' + \
'without any organizations', {
'server_id': self.id,
})
self.pre_start_check()
start_timestamp = utils.now()
response = self.collection.update({
'_id': self.id,
'status': OFFLINE,
'instances_count': 0,
}, {'$set': {
'status': ONLINE,
'start_timestamp': start_timestamp,
}})
if not response['updatedExisting']:
raise ServerInstanceSet('Server instances already running. %r', {
'server_id': self.id,
})
self.status = ONLINE
self.start_timestamp = start_timestamp
replica_count = min(self.replica_count, len(self.hosts))
started_count = 0
error_count = 0
try:
self.publish('start', extra={
'prefered_hosts': host.get_prefered_hosts(
self.hosts, replica_count),
})
for x_timeout in (4, timeout):
for msg in self.subscribe(cursor_id=cursor_id,
timeout=x_timeout):
message = msg['message']
if message == 'started':
started_count += 1
if started_count + error_count >= replica_count:
break
elif message == 'error':
error_count += 1
if started_count + error_count >= replica_count:
break
if started_count:
break
if not started_count:
if error_count:
raise ServerStartError('Server failed to start', {
'server_id': self.id,
})
else:
raise ServerStartError('Server start timed out', {
'server_id': self.id,
})
except:
self.publish('force_stop')
self.collection.update({
'_id': self.id,
}, {'$set': {
'status': OFFLINE,
'instances': [],
'instances_count': 0,
}})
self.status = OFFLINE
self.instances = []
self.instances_count = 0
raise
def task(self):
try:
timestamp_spec = utils.now() - datetime.timedelta(
seconds=settings.vpn.server_ping_ttl)
docs = self.server_collection.find({
'instances.ping_timestamp': {'$lt': timestamp_spec},
}, {
'_id': True,
'instances': True,
})
yield
for doc in docs:
for instance in doc['instances']:
if instance['ping_timestamp'] < timestamp_spec:
self.server_collection.update({
'_id': doc['_id'],
'instances.instance_id': instance['instance_id'],
}, {
'$pull': {
'instances': {
'instance_id': instance['instance_id'],
},
},
'$inc': {
'instances_count': -1,
},
})
yield
docs = self.host_collection.find({
'status': ONLINE,
}, {
'_id': True,
'availability_group': True,
})
yield
hosts_group = {}
for doc in docs:
hosts_group[doc['_id']] = doc.get(
'availability_group', DEFAULT)
yield
response = self.server_collection.aggregate([
{'$match': {
'status': ONLINE,
'start_timestamp': {'$lt': timestamp_spec},
}},
{'$project': {
'_id': True,
'hosts': True,
'instances': True,
'replica_count': True,
'availability_group': True,
'offline_instances_count': {
'$subtract': [
'$replica_count',
'$instances_count',
],
}
}},
{'$match': {
'offline_instances_count': {'$gt': 0},
}},
])
yield
for doc in response:
cur_avail_group = doc.get('availability_group', DEFAULT)
hosts_set = set(doc['hosts'])
group_best = None
group_len_max = 0
server_groups = collections.defaultdict(set)
for hst in hosts_set:
avail_zone = hosts_group.get(hst)
if not avail_zone:
continue
server_groups[avail_zone].add(hst)
group_len = len(server_groups[avail_zone])
if group_len > group_len_max:
group_len_max = group_len
group_best = avail_zone
elif group_len == group_len_max and \
avail_zone == cur_avail_group:
group_best = avail_zone
if cur_avail_group != group_best:
logger.info(
'Rebalancing server availability group',
'server',
server_id=doc['_id'],
current_availability_group=cur_avail_group,
new_availability_group=group_best,
)
self.server_collection.update({
'_id': doc['_id'],
'status': ONLINE,
}, {'$set': {
'instances': [],
'instances_count': 0,
'availability_group': group_best,
}})
messenger.publish('servers', 'rebalance', extra={
'server_id': doc['_id'],
'availability_group': group_best,
})
prefered_hosts = server_groups[group_best]
else:
prefered_hosts = server_groups[cur_avail_group]
active_hosts = set(
[x['host_id'] for x in doc['instances']])
prefered_hosts = list(prefered_hosts - active_hosts)
if not prefered_hosts:
continue
logger.info('Recovering server state', 'server',
server_id=doc['_id'],
prefered_hosts=prefered_hosts,
)
messenger.publish('servers', 'start', extra={
'server_id': doc['_id'],
'send_events': True,
'prefered_hosts': host.get_prefered_hosts(
prefered_hosts, doc['replica_count'])
})
except GeneratorExit:
raise
except:
logger.exception('Error checking server states', 'tasks')
def thread():
platforms = list(DESKTOP_PLATFORMS)
start_timestamp = datetime.datetime(2015, 12, 28, 4, 1, 0)
hosts_collection = mongo.get_collection('hosts')
servers_collection = mongo.get_collection('servers')
clients_collection = mongo.get_collection('clients')
clients_collection.remove({})
for hst in host.iter_hosts():
hosts_collection.update({
'_id': hst.id,
}, {'$set': {
'server_count': 0,
'device_count': 0,
'cpu_usage': 0,
'mem_usage': 0,
'thread_count': 0,
'open_file_count': 0,
'status': ONLINE,
'start_timestamp': start_timestamp,
'ping_timestamp': start_timestamp,
'auto_public_address': None,
'auto_public_address6': None,
'auto_public_host': hst.name + '.pritunl.com',
'auto_public_host6': hst.name + '.pritunl.com',
}})
for svr in server.iter_servers():
prefered_hosts = host.get_prefered_hosts(
svr.hosts, svr.replica_count)
instances = []
for hst in prefered_hosts:
instances.append({
'instance_id': utils.ObjectId(),
'host_id': hst,
'ping_timestamp': utils.now(),
})
servers_collection.update({
'_id': svr.id,
}, {'$set': {
'status': ONLINE,
'pool_cursor': None,
'start_timestamp': start_timestamp,
'availability_group': DEFAULT,
'instances': instances,
'instances_count': len(instances),
}})
for org in svr.iter_orgs():
for usr in org.iter_users():
if usr.type != CERT_CLIENT:
continue
virt_address = svr.get_ip_addr(org.id, usr.id)
virt_address6 = svr.ip4to6(virt_address)
doc = {
'_id': utils.ObjectId(),
'user_id': usr.id,
'server_id': svr.id,
'host_id': settings.local.host_id,
'timestamp': start_timestamp,
'platform': random.choice(platforms),
'type': CERT_CLIENT,
'device_name': utils.random_name(),
'mac_addr': utils.rand_str(16),
'network': svr.network,
'real_address': str(
ipaddress.IPAddress(100000000 + random.randint(
0, 1000000000))),
'virt_address': virt_address,
'virt_address6': virt_address6,
'host_address': settings.local.host.local_addr,
'host_address6': settings.local.host.local_addr6,
'dns_servers': [],
'dns_suffix': None,
'connected_since': int(start_timestamp.strftime('%s')),
}
clients_collection.insert(doc)
for lnk in link.iter_links():
lnk.status = ONLINE
lnk.commit()
for location in lnk.iter_locations():
active = False
for hst in location.iter_hosts():
if not active:
hst.active = True
active = True
hst.status = AVAILABLE
hst.commit(('active', 'status'))
logger.info('Demo initiated', 'demo')
def task(self):
if settings.app.demo_mode:
return
try:
timestamp = utils.now()
timestamp_spec = timestamp - datetime.timedelta(
seconds=settings.vpn.server_ping_ttl)
docs = self.server_collection.find(
{
'instances.ping_timestamp': {
'$lt': timestamp_spec
},
}, {
'_id': True,
'instances': True,
})
yield
for doc in docs:
for instance in doc['instances']:
if instance['ping_timestamp'] < timestamp_spec:
logger.warning(
'Removing instance doc',
'server',
server_id=doc['_id'],
instance_id=instance['instance_id'],
cur_timestamp=timestamp,
ttl_timestamp=timestamp_spec,
ping_timestamp=instance['ping_timestamp'],
)
self.server_collection.update(
{
'_id': doc['_id'],
'instances.instance_id':
instance['instance_id'],
}, {
'$pull': {
'instances': {
'instance_id': instance['instance_id'],
},
},
'$inc': {
'instances_count': -1,
},
})
yield
docs = self.host_collection.find({
'status': ONLINE,
}, {
'_id': True,
'availability_group': True,
})
yield
hosts_group = {}
for doc in docs:
hosts_group[doc['_id']] = doc.get('availability_group',
DEFAULT)
yield
response = self.server_collection.aggregate([
{
'$match': {
'status': ONLINE,
'start_timestamp': {
'$lt': timestamp_spec
},
}
},
{
'$project': {
'_id': True,
'hosts': True,
'instances': True,
'replica_count': True,
'availability_group': True,
'offline_instances_count': {
'$subtract': [
'$replica_count',
'$instances_count',
],
}
}
},
{
'$match': {
'offline_instances_count': {
'$gt': 0
},
}
},
])
yield
recover_count = 0
for doc in response:
cur_avail_group = doc.get('availability_group', DEFAULT)
hosts_set = set(doc['hosts'])
group_best = None
group_len_max = 0
server_groups = collections.defaultdict(set)
for hst in hosts_set:
avail_zone = hosts_group.get(hst)
if not avail_zone:
continue
server_groups[avail_zone].add(hst)
group_len = len(server_groups[avail_zone])
if group_len > group_len_max:
group_len_max = group_len
group_best = avail_zone
elif group_len == group_len_max and \
avail_zone == cur_avail_group:
group_best = avail_zone
if group_best and cur_avail_group != group_best:
logger.info(
'Rebalancing server availability group',
'server',
server_id=doc['_id'],
current_availability_group=cur_avail_group,
new_availability_group=group_best,
)
self.server_collection.update(
{
'_id': doc['_id'],
'status': ONLINE,
}, {
'$set': {
'instances': [],
'instances_count': 0,
'availability_group': group_best,
}
})
messenger.publish('servers',
'rebalance',
extra={
'server_id': doc['_id'],
'availability_group': group_best,
})
prefered_hosts = server_groups[group_best]
else:
prefered_hosts = server_groups[cur_avail_group]
active_hosts = set([x['host_id'] for x in doc['instances']])
prefered_hosts = list(prefered_hosts - active_hosts)
if not prefered_hosts:
continue
if recover_count >= 3:
continue
recover_count += 1
logger.info(
'Recovering server state',
'server',
server_id=doc['_id'],
prefered_hosts=prefered_hosts,
)
messenger.publish('servers',
'start',
extra={
'server_id':
doc['_id'],
'send_events':
True,
'prefered_hosts':
host.get_prefered_hosts(
prefered_hosts, doc['replica_count'])
})
except GeneratorExit:
raise
except:
logger.exception('Error checking server states', 'tasks')
def task(self):
try:
timestamp_spec = utils.now() - datetime.timedelta(
seconds=settings.vpn.server_ping_ttl)
docs = self.server_collection.find({
'instances.ping_timestamp': {'$lt': timestamp_spec},
}, {
'_id': True,
'instances': True,
})
yield
for doc in docs:
for instance in doc['instances']:
if instance['ping_timestamp'] < timestamp_spec:
self.server_collection.update({
'_id': doc['_id'],
'instances.instance_id': instance['instance_id'],
}, {
'$pull': {
'instances': {
'instance_id': instance['instance_id'],
},
},
'$inc': {
'instances_count': -1,
},
})
yield
response = self.server_collection.aggregate([
{'$match': {
'status': ONLINE,
'start_timestamp': {'$lt': timestamp_spec},
}},
{'$project': {
'_id': True,
'hosts': True,
'instances': True,
'replica_count': True,
'offline_instances_count': {
'$subtract': [
'$replica_count',
'$instances_count',
],
}
}},
{'$match': {
'offline_instances_count': {'$gt': 0},
}},
])
yield
for doc in response:
active_hosts = set([x['host_id'] for x in doc['instances']])
hosts = list(set(doc['hosts']) - active_hosts)
if not hosts:
continue
messenger.publish('servers', 'start', extra={
'server_id': doc['_id'],
'send_events': True,
'prefered_hosts': host.get_prefered_hosts(
hosts, doc['replica_count'])
})
except GeneratorExit:
raise
except:
logger.exception('Error checking server states', 'tasks')
def thread():
platforms = list(DESKTOP_PLATFORMS)
start_timestamp = datetime.datetime(2015, 12, 28, 4, 1, 0)
hosts_collection = mongo.get_collection('hosts')
servers_collection = mongo.get_collection('servers')
clients_collection = mongo.get_collection('clients')
clients_collection.remove({})
for hst in host.iter_hosts():
hosts_collection.update({
'_id': hst.id,
}, {
'$set': {
'server_count': 0,
'device_count': 0,
'cpu_usage': 0,
'mem_usage': 0,
'thread_count': 0,
'open_file_count': 0,
'status': ONLINE,
'start_timestamp': start_timestamp,
'ping_timestamp': start_timestamp,
'auto_public_address': None,
'auto_public_address6': None,
'auto_public_host': hst.name + '.pritunl.com',
'auto_public_host6': hst.name + '.pritunl.com',
}
})
for svr in server.iter_servers():
prefered_hosts = host.get_prefered_hosts(svr.hosts,
svr.replica_count)
instances = []
for hst in prefered_hosts:
instances.append({
'instance_id': utils.ObjectId(),
'host_id': hst,
'ping_timestamp': utils.now(),
})
servers_collection.update({
'_id': svr.id,
}, {
'$set': {
'status': ONLINE,
'pool_cursor': None,
'start_timestamp': start_timestamp,
'availability_group': DEFAULT,
'instances': instances,
'instances_count': len(instances),
}
})
for org in svr.iter_orgs():
for usr in org.iter_users():
if usr.type != CERT_CLIENT:
continue
virt_address = svr.get_ip_addr(org.id, usr.id)
virt_address6 = svr.ip4to6(virt_address)
doc = {
'_id':
utils.ObjectId(),
'user_id':
usr.id,
'server_id':
svr.id,
'host_id':
settings.local.host_id,
'timestamp':
start_timestamp,
'platform':
random.choice(platforms),
'type':
CERT_CLIENT,
'device_name':
utils.random_name(),
'mac_addr':
utils.rand_str(16),
'network':
svr.network,
'real_address':
str(
ipaddress.IPAddress(
100000000 + random.randint(0, 1000000000))),
'virt_address':
virt_address,
'virt_address6':
virt_address6,
'host_address':
settings.local.host.local_addr,
'host_address6':
settings.local.host.local_addr6,
'dns_servers': [],
'dns_suffix':
None,
'connected_since':
int(start_timestamp.strftime('%s')),
}
clients_collection.insert(doc)
for lnk in link.iter_links():
lnk.status = ONLINE
lnk.commit()
for location in lnk.iter_locations():
active = False
for hst in location.iter_hosts():
if not active:
hst.active = True
active = True
hst.status = AVAILABLE
hst.commit(('active', 'status'))
logger.info('Demo initiated', 'demo')
def task(self):
try:
timestamp_spec = utils.now() - datetime.timedelta(
seconds=settings.vpn.server_ping_ttl)
docs = self.server_collection.find({
'instances.ping_timestamp': {'$lt': timestamp_spec},
}, {
'_id': True,
'instances': True,
})
yield
for doc in docs:
for instance in doc['instances']:
if instance['ping_timestamp'] < timestamp_spec:
self.server_collection.update({
'_id': doc['_id'],
'instances.instance_id': instance['instance_id'],
}, {
'$pull': {
'instances': {
'instance_id': instance['instance_id'],
},
},
'$inc': {
'instances_count': -1,
},
})
yield
response = self.server_collection.aggregate([
{'$match': {
'status': ONLINE,
'start_timestamp': {'$lt': timestamp_spec},
}},
{'$project': {
'_id': True,
'hosts': True,
'instances': True,
'replica_count': True,
'offline_instances_count': {
'$subtract': [
'$replica_count',
'$instances_count',
],
}
}},
{'$match': {
'offline_instances_count': {'$gt': 0},
}},
])
yield
for doc in response:
active_hosts = set([x['host_id'] for x in doc['instances']])
hosts = list(set(doc['hosts']) - active_hosts)
if not hosts:
continue
messenger.publish('servers', 'start', extra={
'server_id': doc['_id'],
'send_events': True,
'prefered_hosts': host.get_prefered_hosts(
hosts, doc['replica_count'])
})
except GeneratorExit:
raise
except:
logger.exception('Error checking server states', 'tasks')
yield interrupter_sleep(settings.vpn.server_ping)
def start(self, timeout=None):
timeout = timeout or settings.vpn.op_timeout
cursor_id = self.get_cursor_id()
if self.status != OFFLINE:
return
if not self.dh_params:
self.generate_dh_param()
return
if not self.organizations:
raise ServerMissingOrg("Server cannot be started " + "without any organizations", {"server_id": self.id})
self.pre_start_check()
start_timestamp = utils.now()
response = self.collection.update(
{"_id": self.id, "status": OFFLINE, "instances_count": 0},
{"$set": {"status": ONLINE, "start_timestamp": start_timestamp}},
)
if not response["updatedExisting"]:
raise ServerInstanceSet("Server instances already running. %r", {"server_id": self.id})
self.status = ONLINE
self.start_timestamp = start_timestamp
replica_count = min(self.replica_count, len(self.hosts))
started_count = 0
error_count = 0
try:
self.publish("start", extra={"prefered_hosts": host.get_prefered_hosts(self.hosts, replica_count)})
for x_timeout in (4, timeout):
for msg in self.subscribe(cursor_id=cursor_id, timeout=x_timeout):
message = msg["message"]
if message == "started":
started_count += 1
if started_count + error_count >= replica_count:
break
elif message == "error":
error_count += 1
if started_count + error_count >= replica_count:
break
if started_count:
break
if not started_count:
if error_count:
raise ServerStartError("Server failed to start", {"server_id": self.id})
else:
raise ServerStartError("Server start timed out", {"server_id": self.id})
except:
self.publish("force_stop")
self.collection.update(
{"_id": self.id}, {"$set": {"status": OFFLINE, "instances": [], "instances_count": 0}}
)
self.status = OFFLINE
self.instances = []
self.instances_count = 0
raise
