def setup_mongo():
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
read_pref = _get_read_pref(settings.conf.mongodb_read_preference)
if read_pref:
client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
read_preference=read_pref,
)
else:
client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
settings_col = getattr(database, prefix + 'settings')
app_settings = settings_col.find_one({'_id': 'app'})
if app_settings:
secondary_mongodb_uri = app_settings.get('secondary_mongodb_uri')
else:
secondary_mongodb_uri = None
if secondary_mongodb_uri:
while True:
try:
read_pref = _get_read_pref(
settings.conf.mongodb_read_preference)
if read_pref:
secondary_client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
read_preference=read_pref,
)
else:
secondary_client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception(
'Error connecting to secondary mongodb server')
secondary_database = secondary_client.get_default_database()
else:
secondary_database = database
cur_collections = secondary_database.collection_names()
if prefix + 'messages' not in cur_collections:
secondary_database.create_collection(prefix + 'messages',
capped=True,
size=5000192,
max=1500)
mongo.collections.update({
'transaction':
getattr(database, prefix + 'transaction'),
'queue':
getattr(database, prefix + 'queue'),
'tasks':
getattr(database, prefix + 'tasks'),
'settings':
getattr(database, prefix + 'settings'),
'messages':
getattr(secondary_database, prefix + 'messages'),
'administrators':
getattr(database, prefix + 'administrators'),
'users':
getattr(database, prefix + 'users'),
'users_audit':
getattr(database, prefix + 'users_audit'),
'users_key_link':
getattr(secondary_database, prefix + 'users_key_link'),
'users_net_link':
getattr(database, prefix + 'users_net_link'),
'clients':
getattr(database, prefix + 'clients'),
'clients_pool':
getattr(database, prefix + 'clients_pool'),
'organizations':
getattr(database, prefix + 'organizations'),
'hosts':
getattr(database, prefix + 'hosts'),
'hosts_usage':
getattr(database, prefix + 'hosts_usage'),
'servers':
getattr(database, prefix + 'servers'),
'servers_output':
getattr(database, prefix + 'servers_output'),
'servers_output_link':
getattr(database, prefix + 'servers_output_link'),
'servers_bandwidth':
getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool':
getattr(database, prefix + 'servers_ip_pool'),
'routes_reserve':
getattr(database, prefix + 'routes_reserve'),
'dh_params':
getattr(database, prefix + 'dh_params'),
'auth_sessions':
getattr(secondary_database, prefix + 'auth_sessions'),
'auth_csrf_tokens':
getattr(secondary_database, prefix + 'auth_csrf_tokens'),
'auth_nonces':
getattr(secondary_database, prefix + 'auth_nonces'),
'auth_limiter':
getattr(secondary_database, prefix + 'auth_limiter'),
'otp':
getattr(secondary_database, prefix + 'otp'),
'otp_cache':
getattr(secondary_database, prefix + 'otp_cache'),
'sso_tokens':
getattr(secondary_database, prefix + 'sso_tokens'),
'sso_cache':
getattr(secondary_database, prefix + 'sso_cache'),
'vxlans':
getattr(database, prefix + 'vxlans'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
settings.local.mongo_time = None
while True:
try:
utils.sync_time()
break
except:
logger.exception('Failed to sync time', 'setup')
time.sleep(30)
settings.init()
cur_collections = database.collection_names()
if prefix + 'logs' not in cur_collections:
log_limit = settings.app.log_limit
database.create_collection(prefix + 'logs',
capped=True,
size=log_limit * 1024,
max=log_limit)
if prefix + 'log_entries' not in cur_collections:
log_entry_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries',
capped=True,
size=log_entry_limit * 512,
max=log_entry_limit)
mongo.collections.update({
'logs':
getattr(database, prefix + 'logs'),
'log_entries':
getattr(database, prefix + 'log_entries'),
})
mongo.collections['logs'].name_str = 'logs'
mongo.collections['log_entries'].name_str = 'log_entries'
upsert_index(mongo.collections['logs'], 'timestamp', background=True)
upsert_index(mongo.collections['transaction'],
'lock_id',
background=True,
unique=True)
upsert_index(mongo.collections['transaction'], [
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
],
background=True)
upsert_index(mongo.collections['queue'], 'runner_id', background=True)
upsert_index(mongo.collections['queue'], 'ttl_timestamp', background=True)
upsert_index(mongo.collections['tasks'], [
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['log_entries'], [
('timestamp', pymongo.DESCENDING),
],
background=True)
upsert_index(mongo.collections['messages'], 'channel', background=True)
upsert_index(mongo.collections['administrators'],
'username',
background=True,
unique=True)
upsert_index(mongo.collections['users'], 'resource_id', background=True)
upsert_index(mongo.collections['users'], [
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['users'], [
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['users_audit'], [
('org_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['users_audit'], [
('timestamp', pymongo.DESCENDING),
],
background=True)
upsert_index(mongo.collections['users_key_link'],
'key_id',
background=True)
upsert_index(mongo.collections['users_key_link'],
'short_id',
background=True,
unique=True)
upsert_index(mongo.collections['users_net_link'],
'user_id',
background=True)
upsert_index(mongo.collections['users_net_link'],
'org_id',
background=True)
upsert_index(mongo.collections['users_net_link'],
'network',
background=True)
upsert_index(mongo.collections['clients'], 'user_id', background=True)
upsert_index(mongo.collections['clients'], 'domain', background=True)
upsert_index(mongo.collections['clients'],
'virt_address_num',
background=True)
upsert_index(mongo.collections['clients'], [
('server_id', pymongo.ASCENDING),
('type', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['clients'], [
('host_id', pymongo.ASCENDING),
('type', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['clients_pool'],
'client_id',
background=True)
upsert_index(mongo.collections['clients_pool'],
'timestamp',
background=True)
upsert_index(mongo.collections['clients_pool'], [
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['organizations'], 'type', background=True)
upsert_index(mongo.collections['organizations'],
'auth_token',
background=True)
upsert_index(mongo.collections['hosts'], 'name', background=True)
upsert_index(mongo.collections['hosts_usage'], [
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['servers'], 'name', background=True)
upsert_index(mongo.collections['servers'],
'ping_timestamp',
background=True)
upsert_index(mongo.collections['servers_output'], [
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['servers_output_link'], [
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['servers_bandwidth'], [
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['servers_ip_pool'], [
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['servers_ip_pool'], [
('server_id', pymongo.ASCENDING),
('_id', pymongo.DESCENDING),
],
background=True)
upsert_index(mongo.collections['servers_ip_pool'],
'user_id',
background=True)
upsert_index(mongo.collections['routes_reserve'],
'timestamp',
background=True)
upsert_index(mongo.collections['dh_params'],
'dh_param_bits',
background=True)
upsert_index(mongo.collections['auth_nonces'], [
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
],
background=True,
unique=True)
upsert_index(mongo.collections['sso_cache'], [
('user_id', pymongo.ASCENDING),
('server_id', pymongo.ASCENDING),
('remote_ip', pymongo.ASCENDING),
('mac_addr', pymongo.ASCENDING),
('platform', pymongo.ASCENDING),
('device_name', pymongo.ASCENDING),
],
background=True)
upsert_index(mongo.collections['vxlans'],
'server_id',
background=True,
unique=True)
upsert_index(mongo.collections['tasks'],
'timestamp',
background=True,
expireAfterSeconds=300)
upsert_index(mongo.collections['clients'],
'timestamp',
background=True,
expireAfterSeconds=settings.vpn.client_ttl)
upsert_index(mongo.collections['users_key_link'],
'timestamp',
background=True,
expireAfterSeconds=settings.app.key_link_timeout)
upsert_index(mongo.collections['auth_sessions'],
'timestamp',
background=True,
expireAfterSeconds=settings.app.session_timeout)
upsert_index(mongo.collections['auth_nonces'],
'timestamp',
background=True,
expireAfterSeconds=settings.app.auth_time_window * 2.1)
upsert_index(mongo.collections['auth_limiter'],
'timestamp',
background=True,
expireAfterSeconds=settings.app.auth_limiter_ttl)
upsert_index(mongo.collections['otp'],
'timestamp',
background=True,
expireAfterSeconds=120)
upsert_index(mongo.collections['otp_cache'],
'timestamp',
background=True,
expireAfterSeconds=settings.user.otp_cache_ttl)
upsert_index(mongo.collections['sso_tokens'],
'timestamp',
background=True,
expireAfterSeconds=600)
upsert_index(mongo.collections['sso_cache'],
'timestamp',
background=True,
expireAfterSeconds=settings.app.sso_cache_timeout)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
secret_key2 = settings.app.cookie_secret2
settings_commit = False
if not secret_key:
settings_commit = True
secret_key = utils.rand_str(64)
settings.app.cookie_secret = secret_key
if not secret_key2:
settings_commit = True
settings.app.cookie_secret2 = utils.rand_str(64)
if settings_commit:
settings.commit()
app.app.secret_key = secret_key.encode()
def setup_mongo():
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
read_pref = _get_read_pref(settings.conf.mongodb_read_preference)
if read_pref:
client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT,
read_preference=read_pref,
)
else:
client = pymongo.MongoClient(
settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT,
)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
settings_col = getattr(database, prefix + 'settings')
app_settings = settings_col.find_one({'_id': 'app'})
if app_settings:
secondary_mongodb_uri = app_settings.get('secondary_mongodb_uri')
else:
secondary_mongodb_uri = None
if secondary_mongodb_uri:
while True:
try:
read_pref = _get_read_pref(
settings.conf.mongodb_read_preference)
if read_pref:
secondary_client = pymongo.MongoClient(
secondary_mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT,
read_preference=read_pref,
)
else:
secondary_client = pymongo.MongoClient(
secondary_mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT,
socketTimeoutMS=MONGO_SOCKET_TIMEOUT,
serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT,
)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception(
'Error connecting to secondary mongodb server')
secondary_database = secondary_client.get_default_database()
else:
secondary_database = database
mongo.database = database
mongo.secondary_database = secondary_database
cur_collections = database.collection_names()
cur_sec_collections = secondary_database.collection_names()
if 'authorities' in cur_collections or \
'authorities' in cur_sec_collections:
raise TypeError('Cannot connect to a Pritunl Zero database')
mongo.collection_types = {
'transaction': 1,
'queue': 1,
'tasks': 1,
'settings': 1,
'messages': 2,
'administrators': 1,
'users': 1,
'users_audit': 1,
'users_key_link': 2,
'users_net_link': 1,
'clients': 1,
'clients_pool': 1,
'organizations': 1,
'hosts': 1,
'hosts_usage': 1,
'servers': 1,
'servers_output': 1,
'servers_output_link': 1,
'servers_bandwidth': 1,
'servers_ip_pool': 1,
'links': 1,
'links_locations': 1,
'links_hosts': 1,
'routes_reserve': 1,
'dh_params': 1,
'acme_challenges': 1,
'auth_sessions': 2,
'auth_csrf_tokens': 2,
'auth_nonces': 2,
'auth_limiter': 2,
'otp': 2,
'otp_cache': 2,
'yubikey': 2,
'sso_tokens': 2,
'sso_push_cache': 2,
'sso_client_cache': 2,
'sso_passcode_cache': 2,
'vxlans': 1,
'logs': 1,
'log_entries': 1,
}
cur_collections = mongo.secondary_database.collection_names()
if prefix + 'messages' not in cur_collections:
mongo.secondary_database.create_collection(
prefix + 'messages', capped=True,
size=5000192, max=1000)
elif not mongo.get_collection('messages').options().get('capped'):
mongo.get_collection('messages').drop()
mongo.secondary_database.create_collection(
prefix + 'messages', capped=True,
size=5000192, max=1000)
settings.local.mongo_time = None
while True:
try:
utils.sync_time()
break
except:
logger.exception('Failed to sync time', 'setup')
time.sleep(30)
settings.init()
upsert_indexes()
if not auth.Administrator.collection.find_one():
default_admin = auth.Administrator(
username=DEFAULT_USERNAME,
)
default_admin.generate_default_password()
default_admin.commit()
secret_key = settings.app.cookie_secret
secret_key2 = settings.app.cookie_secret2
settings_commit = False
if not secret_key:
settings_commit = True
secret_key = utils.rand_str(64)
settings.app.cookie_secret = secret_key
if not secret_key2:
settings_commit = True
settings.app.cookie_secret2 = utils.rand_str(64)
if settings_commit:
settings.commit()
app.app.secret_key = secret_key.encode()
def setup_mongo():
prefix = settings.conf.mongodb_collection_prefix or ''
last_error = time.time() - 24
while True:
try:
client = pymongo.MongoClient(settings.conf.mongodb_uri,
connectTimeoutMS=MONGO_CONNECT_TIMEOUT)
break
except pymongo.errors.ConnectionFailure:
time.sleep(0.5)
if time.time() - last_error > 30:
last_error = time.time()
logger.exception('Error connecting to mongodb server')
database = client.get_default_database()
cur_collections = database.collection_names()
if prefix + 'messages' not in cur_collections:
database.create_collection(prefix + 'messages', capped=True,
size=100000, max=1024)
mongo.collections.update({
'time_sync': getattr(database, prefix + 'time_sync'),
'transaction': getattr(database, prefix + 'transaction'),
'queue': getattr(database, prefix + 'queue'),
'task': getattr(database, prefix + 'task'),
'settings': getattr(database, prefix + 'settings'),
'messages': getattr(database, prefix + 'messages'),
'administrators': getattr(database, prefix + 'administrators'),
'users': getattr(database, prefix + 'users'),
'users_key_link': getattr(database, prefix + 'users_key_link'),
'organizations': getattr(database, prefix + 'organizations'),
'hosts': getattr(database, prefix + 'hosts'),
'hosts_usage': getattr(database, prefix + 'hosts_usage'),
'servers': getattr(database, prefix + 'servers'),
'servers_output': getattr(database, prefix + 'servers_output'),
'servers_output_link': getattr(database,
prefix + 'servers_output_link'),
'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'),
'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'),
'dh_params': getattr(database, prefix + 'dh_params'),
'auth_sessions': getattr(database, prefix + 'auth_sessions'),
'auth_nonces': getattr(database, prefix + 'auth_nonces'),
'auth_limiter': getattr(database, prefix + 'auth_limiter'),
'otp': getattr(database, prefix + 'otp'),
'otp_cache': getattr(database, prefix + 'otp_cache'),
})
for collection_name, collection in mongo.collections.items():
collection.name_str = collection_name
utils.sync_time()
settings.init()
if prefix + 'logs' not in cur_collections:
log_limit = settings.app.log_limit
database.create_collection(prefix + 'logs', capped=True,
size=log_limit * 1024, max=log_limit)
if prefix + 'log_entries' not in cur_collections:
log_entry_limit = settings.app.log_entry_limit
database.create_collection(prefix + 'log_entries', capped=True,
size=log_entry_limit * 512, max=log_entry_limit)
mongo.collections.update({
'logs': getattr(database, prefix + 'logs'),
'log_entries': getattr(database, prefix + 'log_entries'),
})
mongo.collections['logs'].name_str = 'logs'
mongo.collections['log_entries'].name_str = 'log_entries'
mongo.collections['logs'].ensure_index('timestamp', background=True)
mongo.collections['transaction'].ensure_index('lock_id',
background=True, unique=True)
mongo.collections['transaction'].ensure_index([
('ttl_timestamp', pymongo.ASCENDING),
('state', pymongo.ASCENDING),
('priority', pymongo.DESCENDING),
], background=True)
mongo.collections['queue'].ensure_index('runner_id', background=True)
mongo.collections['queue'].ensure_index('ttl_timestamp', background=True)
mongo.collections['task'].ensure_index('type', background=True,
unique=True)
mongo.collections['task'].ensure_index('ttl_timestamp', background=True)
mongo.collections['log_entries'].ensure_index([
('timestamp', pymongo.DESCENDING),
], background=True)
mongo.collections['messages'].ensure_index('channel', background=True)
mongo.collections['administrators'].ensure_index('username',
background=True, unique=True)
mongo.collections['users'].ensure_index('resource_id', background=True)
mongo.collections['users'].ensure_index([
('type', pymongo.ASCENDING),
('org_id', pymongo.ASCENDING),
], background=True)
mongo.collections['users'].ensure_index([
('org_id', pymongo.ASCENDING),
('name', pymongo.ASCENDING),
], background=True)
mongo.collections['users_key_link'].ensure_index('key_id', background=True)
mongo.collections['users_key_link'].ensure_index('short_id',
background=True, unique=True)
mongo.collections['organizations'].ensure_index('type', background=True)
mongo.collections['hosts'].ensure_index('name', background=True)
mongo.collections['hosts_usage'].ensure_index([
('host_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
], background=True)
mongo.collections['servers'].ensure_index('name', background=True)
mongo.collections['servers'].ensure_index('ping_timestamp',
background=True)
mongo.collections['servers_output'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
], background=True)
mongo.collections['servers_output_link'].ensure_index([
('server_id', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
], background=True)
mongo.collections['servers_bandwidth'].ensure_index([
('server_id', pymongo.ASCENDING),
('period', pymongo.ASCENDING),
('timestamp', pymongo.ASCENDING),
], background=True)
mongo.collections['servers_ip_pool'].ensure_index([
('server_id', pymongo.ASCENDING),
('user_id', pymongo.ASCENDING),
], background=True)
mongo.collections['servers_ip_pool'].ensure_index('user_id',
background=True)
mongo.collections['dh_params'].ensure_index('dh_param_bits',
background=True)
mongo.collections['auth_nonces'].ensure_index([
('token', pymongo.ASCENDING),
('nonce', pymongo.ASCENDING),
], background=True, unique=True)
mongo.collections['users_key_link'].ensure_index('timestamp',
background=True, expireAfterSeconds=settings.app.key_link_timeout)
mongo.collections['auth_sessions'].ensure_index('timestamp',
background=True, expireAfterSeconds=settings.app.session_timeout)
mongo.collections['auth_nonces'].ensure_index('timestamp', background=True,
expireAfterSeconds=settings.app.auth_time_window * 2.1)
mongo.collections['auth_limiter'].ensure_index('timestamp',
background=True, expireAfterSeconds=settings.app.auth_limiter_ttl)
mongo.collections['otp'].ensure_index('timestamp', background=True,
expireAfterSeconds=120)
mongo.collections['otp_cache'].ensure_index('timestamp', background=True,
expireAfterSeconds=settings.user.otp_cache_ttl)
if not auth.Administrator.collection.find_one():
auth.Administrator(
username=DEFAULT_USERNAME,
password=DEFAULT_PASSWORD,
default=True,
).commit()
secret_key = settings.app.cookie_secret
if not secret_key:
secret_key = re.sub(r'[\W_]+', '',
base64.b64encode(os.urandom(128)))[:64]
settings.app.cookie_secret = secret_key
settings.commit()
app.app.secret_key = secret_key.encode()
