def setup_mongo(): prefix = settings.conf.mongodb_collection_prefix or '' last_error = time.time() - 24 while True: try: read_pref = _get_read_pref(settings.conf.mongodb_read_preference) if read_pref: client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, read_preference=read_pref, ) else: client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, ) break except pymongo.errors.ConnectionFailure: time.sleep(0.5) if time.time() - last_error > 30: last_error = time.time() logger.exception('Error connecting to mongodb server') database = client.get_default_database() settings_col = getattr(database, prefix + 'settings') app_settings = settings_col.find_one({'_id': 'app'}) if app_settings: secondary_mongodb_uri = app_settings.get('secondary_mongodb_uri') else: secondary_mongodb_uri = None if secondary_mongodb_uri: while True: try: read_pref = _get_read_pref( settings.conf.mongodb_read_preference) if read_pref: secondary_client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, read_preference=read_pref, ) else: secondary_client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, ) break except pymongo.errors.ConnectionFailure: time.sleep(0.5) if time.time() - last_error > 30: last_error = time.time() logger.exception( 'Error connecting to secondary mongodb server') secondary_database = secondary_client.get_default_database() else: secondary_database = database cur_collections = secondary_database.collection_names() if prefix + 'messages' not in cur_collections: secondary_database.create_collection(prefix + 'messages', capped=True, size=5000192, max=1500) mongo.collections.update({ 'transaction': getattr(database, prefix + 'transaction'), 'queue': getattr(database, prefix + 'queue'), 'tasks': getattr(database, prefix + 'tasks'), 'settings': getattr(database, prefix + 'settings'), 'messages': getattr(secondary_database, prefix + 'messages'), 'administrators': getattr(database, prefix + 'administrators'), 'users': getattr(database, prefix + 'users'), 'users_audit': getattr(database, prefix + 'users_audit'), 'users_key_link': getattr(secondary_database, prefix + 'users_key_link'), 'users_net_link': getattr(database, prefix + 'users_net_link'), 'clients': getattr(database, prefix + 'clients'), 'clients_pool': getattr(database, prefix + 'clients_pool'), 'organizations': getattr(database, prefix + 'organizations'), 'hosts': getattr(database, prefix + 'hosts'), 'hosts_usage': getattr(database, prefix + 'hosts_usage'), 'servers': getattr(database, prefix + 'servers'), 'servers_output': getattr(database, prefix + 'servers_output'), 'servers_output_link': getattr(database, prefix + 'servers_output_link'), 'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'), 'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'), 'routes_reserve': getattr(database, prefix + 'routes_reserve'), 'dh_params': getattr(database, prefix + 'dh_params'), 'auth_sessions': getattr(secondary_database, prefix + 'auth_sessions'), 'auth_csrf_tokens': getattr(secondary_database, prefix + 'auth_csrf_tokens'), 'auth_nonces': getattr(secondary_database, prefix + 'auth_nonces'), 'auth_limiter': getattr(secondary_database, prefix + 'auth_limiter'), 'otp': getattr(secondary_database, prefix + 'otp'), 'otp_cache': getattr(secondary_database, prefix + 'otp_cache'), 'sso_tokens': getattr(secondary_database, prefix + 'sso_tokens'), 'sso_cache': getattr(secondary_database, prefix + 'sso_cache'), 'vxlans': getattr(database, prefix + 'vxlans'), }) for collection_name, collection in mongo.collections.items(): collection.name_str = collection_name settings.local.mongo_time = None while True: try: utils.sync_time() break except: logger.exception('Failed to sync time', 'setup') time.sleep(30) settings.init() cur_collections = database.collection_names() if prefix + 'logs' not in cur_collections: log_limit = settings.app.log_limit database.create_collection(prefix + 'logs', capped=True, size=log_limit * 1024, max=log_limit) if prefix + 'log_entries' not in cur_collections: log_entry_limit = settings.app.log_entry_limit database.create_collection(prefix + 'log_entries', capped=True, size=log_entry_limit * 512, max=log_entry_limit) mongo.collections.update({ 'logs': getattr(database, prefix + 'logs'), 'log_entries': getattr(database, prefix + 'log_entries'), }) mongo.collections['logs'].name_str = 'logs' mongo.collections['log_entries'].name_str = 'log_entries' upsert_index(mongo.collections['logs'], 'timestamp', background=True) upsert_index(mongo.collections['transaction'], 'lock_id', background=True, unique=True) upsert_index(mongo.collections['transaction'], [ ('ttl_timestamp', pymongo.ASCENDING), ('state', pymongo.ASCENDING), ('priority', pymongo.DESCENDING), ], background=True) upsert_index(mongo.collections['queue'], 'runner_id', background=True) upsert_index(mongo.collections['queue'], 'ttl_timestamp', background=True) upsert_index(mongo.collections['tasks'], [ ('ttl_timestamp', pymongo.ASCENDING), ('state', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['log_entries'], [ ('timestamp', pymongo.DESCENDING), ], background=True) upsert_index(mongo.collections['messages'], 'channel', background=True) upsert_index(mongo.collections['administrators'], 'username', background=True, unique=True) upsert_index(mongo.collections['users'], 'resource_id', background=True) upsert_index(mongo.collections['users'], [ ('type', pymongo.ASCENDING), ('org_id', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['users'], [ ('org_id', pymongo.ASCENDING), ('name', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['users_audit'], [ ('org_id', pymongo.ASCENDING), ('user_id', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['users_audit'], [ ('timestamp', pymongo.DESCENDING), ], background=True) upsert_index(mongo.collections['users_key_link'], 'key_id', background=True) upsert_index(mongo.collections['users_key_link'], 'short_id', background=True, unique=True) upsert_index(mongo.collections['users_net_link'], 'user_id', background=True) upsert_index(mongo.collections['users_net_link'], 'org_id', background=True) upsert_index(mongo.collections['users_net_link'], 'network', background=True) upsert_index(mongo.collections['clients'], 'user_id', background=True) upsert_index(mongo.collections['clients'], 'domain', background=True) upsert_index(mongo.collections['clients'], 'virt_address_num', background=True) upsert_index(mongo.collections['clients'], [ ('server_id', pymongo.ASCENDING), ('type', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['clients'], [ ('host_id', pymongo.ASCENDING), ('type', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['clients_pool'], 'client_id', background=True) upsert_index(mongo.collections['clients_pool'], 'timestamp', background=True) upsert_index(mongo.collections['clients_pool'], [ ('server_id', pymongo.ASCENDING), ('user_id', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['organizations'], 'type', background=True) upsert_index(mongo.collections['organizations'], 'auth_token', background=True) upsert_index(mongo.collections['hosts'], 'name', background=True) upsert_index(mongo.collections['hosts_usage'], [ ('host_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['servers'], 'name', background=True) upsert_index(mongo.collections['servers'], 'ping_timestamp', background=True) upsert_index(mongo.collections['servers_output'], [ ('server_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['servers_output_link'], [ ('server_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['servers_bandwidth'], [ ('server_id', pymongo.ASCENDING), ('period', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['servers_ip_pool'], [ ('server_id', pymongo.ASCENDING), ('user_id', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['servers_ip_pool'], [ ('server_id', pymongo.ASCENDING), ('_id', pymongo.DESCENDING), ], background=True) upsert_index(mongo.collections['servers_ip_pool'], 'user_id', background=True) upsert_index(mongo.collections['routes_reserve'], 'timestamp', background=True) upsert_index(mongo.collections['dh_params'], 'dh_param_bits', background=True) upsert_index(mongo.collections['auth_nonces'], [ ('token', pymongo.ASCENDING), ('nonce', pymongo.ASCENDING), ], background=True, unique=True) upsert_index(mongo.collections['sso_cache'], [ ('user_id', pymongo.ASCENDING), ('server_id', pymongo.ASCENDING), ('remote_ip', pymongo.ASCENDING), ('mac_addr', pymongo.ASCENDING), ('platform', pymongo.ASCENDING), ('device_name', pymongo.ASCENDING), ], background=True) upsert_index(mongo.collections['vxlans'], 'server_id', background=True, unique=True) upsert_index(mongo.collections['tasks'], 'timestamp', background=True, expireAfterSeconds=300) upsert_index(mongo.collections['clients'], 'timestamp', background=True, expireAfterSeconds=settings.vpn.client_ttl) upsert_index(mongo.collections['users_key_link'], 'timestamp', background=True, expireAfterSeconds=settings.app.key_link_timeout) upsert_index(mongo.collections['auth_sessions'], 'timestamp', background=True, expireAfterSeconds=settings.app.session_timeout) upsert_index(mongo.collections['auth_nonces'], 'timestamp', background=True, expireAfterSeconds=settings.app.auth_time_window * 2.1) upsert_index(mongo.collections['auth_limiter'], 'timestamp', background=True, expireAfterSeconds=settings.app.auth_limiter_ttl) upsert_index(mongo.collections['otp'], 'timestamp', background=True, expireAfterSeconds=120) upsert_index(mongo.collections['otp_cache'], 'timestamp', background=True, expireAfterSeconds=settings.user.otp_cache_ttl) upsert_index(mongo.collections['sso_tokens'], 'timestamp', background=True, expireAfterSeconds=600) upsert_index(mongo.collections['sso_cache'], 'timestamp', background=True, expireAfterSeconds=settings.app.sso_cache_timeout) if not auth.Administrator.collection.find_one(): auth.Administrator( username=DEFAULT_USERNAME, password=DEFAULT_PASSWORD, default=True, ).commit() secret_key = settings.app.cookie_secret secret_key2 = settings.app.cookie_secret2 settings_commit = False if not secret_key: settings_commit = True secret_key = utils.rand_str(64) settings.app.cookie_secret = secret_key if not secret_key2: settings_commit = True settings.app.cookie_secret2 = utils.rand_str(64) if settings_commit: settings.commit() app.app.secret_key = secret_key.encode()
def setup_mongo(): prefix = settings.conf.mongodb_collection_prefix or '' last_error = time.time() - 24 while True: try: read_pref = _get_read_pref(settings.conf.mongodb_read_preference) if read_pref: client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT, read_preference=read_pref, ) else: client = pymongo.MongoClient( settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT, ) break except pymongo.errors.ConnectionFailure: time.sleep(0.5) if time.time() - last_error > 30: last_error = time.time() logger.exception('Error connecting to mongodb server') database = client.get_default_database() settings_col = getattr(database, prefix + 'settings') app_settings = settings_col.find_one({'_id': 'app'}) if app_settings: secondary_mongodb_uri = app_settings.get('secondary_mongodb_uri') else: secondary_mongodb_uri = None if secondary_mongodb_uri: while True: try: read_pref = _get_read_pref( settings.conf.mongodb_read_preference) if read_pref: secondary_client = pymongo.MongoClient( secondary_mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT, read_preference=read_pref, ) else: secondary_client = pymongo.MongoClient( secondary_mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT, socketTimeoutMS=MONGO_SOCKET_TIMEOUT, serverSelectionTimeoutMS=MONGO_SOCKET_TIMEOUT, ) break except pymongo.errors.ConnectionFailure: time.sleep(0.5) if time.time() - last_error > 30: last_error = time.time() logger.exception( 'Error connecting to secondary mongodb server') secondary_database = secondary_client.get_default_database() else: secondary_database = database mongo.database = database mongo.secondary_database = secondary_database cur_collections = database.collection_names() cur_sec_collections = secondary_database.collection_names() if 'authorities' in cur_collections or \ 'authorities' in cur_sec_collections: raise TypeError('Cannot connect to a Pritunl Zero database') mongo.collection_types = { 'transaction': 1, 'queue': 1, 'tasks': 1, 'settings': 1, 'messages': 2, 'administrators': 1, 'users': 1, 'users_audit': 1, 'users_key_link': 2, 'users_net_link': 1, 'clients': 1, 'clients_pool': 1, 'organizations': 1, 'hosts': 1, 'hosts_usage': 1, 'servers': 1, 'servers_output': 1, 'servers_output_link': 1, 'servers_bandwidth': 1, 'servers_ip_pool': 1, 'links': 1, 'links_locations': 1, 'links_hosts': 1, 'routes_reserve': 1, 'dh_params': 1, 'acme_challenges': 1, 'auth_sessions': 2, 'auth_csrf_tokens': 2, 'auth_nonces': 2, 'auth_limiter': 2, 'otp': 2, 'otp_cache': 2, 'yubikey': 2, 'sso_tokens': 2, 'sso_push_cache': 2, 'sso_client_cache': 2, 'sso_passcode_cache': 2, 'vxlans': 1, 'logs': 1, 'log_entries': 1, } cur_collections = mongo.secondary_database.collection_names() if prefix + 'messages' not in cur_collections: mongo.secondary_database.create_collection( prefix + 'messages', capped=True, size=5000192, max=1000) elif not mongo.get_collection('messages').options().get('capped'): mongo.get_collection('messages').drop() mongo.secondary_database.create_collection( prefix + 'messages', capped=True, size=5000192, max=1000) settings.local.mongo_time = None while True: try: utils.sync_time() break except: logger.exception('Failed to sync time', 'setup') time.sleep(30) settings.init() upsert_indexes() if not auth.Administrator.collection.find_one(): default_admin = auth.Administrator( username=DEFAULT_USERNAME, ) default_admin.generate_default_password() default_admin.commit() secret_key = settings.app.cookie_secret secret_key2 = settings.app.cookie_secret2 settings_commit = False if not secret_key: settings_commit = True secret_key = utils.rand_str(64) settings.app.cookie_secret = secret_key if not secret_key2: settings_commit = True settings.app.cookie_secret2 = utils.rand_str(64) if settings_commit: settings.commit() app.app.secret_key = secret_key.encode()
def setup_mongo(): prefix = settings.conf.mongodb_collection_prefix or '' last_error = time.time() - 24 while True: try: client = pymongo.MongoClient(settings.conf.mongodb_uri, connectTimeoutMS=MONGO_CONNECT_TIMEOUT) break except pymongo.errors.ConnectionFailure: time.sleep(0.5) if time.time() - last_error > 30: last_error = time.time() logger.exception('Error connecting to mongodb server') database = client.get_default_database() cur_collections = database.collection_names() if prefix + 'messages' not in cur_collections: database.create_collection(prefix + 'messages', capped=True, size=100000, max=1024) mongo.collections.update({ 'time_sync': getattr(database, prefix + 'time_sync'), 'transaction': getattr(database, prefix + 'transaction'), 'queue': getattr(database, prefix + 'queue'), 'task': getattr(database, prefix + 'task'), 'settings': getattr(database, prefix + 'settings'), 'messages': getattr(database, prefix + 'messages'), 'administrators': getattr(database, prefix + 'administrators'), 'users': getattr(database, prefix + 'users'), 'users_key_link': getattr(database, prefix + 'users_key_link'), 'organizations': getattr(database, prefix + 'organizations'), 'hosts': getattr(database, prefix + 'hosts'), 'hosts_usage': getattr(database, prefix + 'hosts_usage'), 'servers': getattr(database, prefix + 'servers'), 'servers_output': getattr(database, prefix + 'servers_output'), 'servers_output_link': getattr(database, prefix + 'servers_output_link'), 'servers_bandwidth': getattr(database, prefix + 'servers_bandwidth'), 'servers_ip_pool': getattr(database, prefix + 'servers_ip_pool'), 'dh_params': getattr(database, prefix + 'dh_params'), 'auth_sessions': getattr(database, prefix + 'auth_sessions'), 'auth_nonces': getattr(database, prefix + 'auth_nonces'), 'auth_limiter': getattr(database, prefix + 'auth_limiter'), 'otp': getattr(database, prefix + 'otp'), 'otp_cache': getattr(database, prefix + 'otp_cache'), }) for collection_name, collection in mongo.collections.items(): collection.name_str = collection_name utils.sync_time() settings.init() if prefix + 'logs' not in cur_collections: log_limit = settings.app.log_limit database.create_collection(prefix + 'logs', capped=True, size=log_limit * 1024, max=log_limit) if prefix + 'log_entries' not in cur_collections: log_entry_limit = settings.app.log_entry_limit database.create_collection(prefix + 'log_entries', capped=True, size=log_entry_limit * 512, max=log_entry_limit) mongo.collections.update({ 'logs': getattr(database, prefix + 'logs'), 'log_entries': getattr(database, prefix + 'log_entries'), }) mongo.collections['logs'].name_str = 'logs' mongo.collections['log_entries'].name_str = 'log_entries' mongo.collections['logs'].ensure_index('timestamp', background=True) mongo.collections['transaction'].ensure_index('lock_id', background=True, unique=True) mongo.collections['transaction'].ensure_index([ ('ttl_timestamp', pymongo.ASCENDING), ('state', pymongo.ASCENDING), ('priority', pymongo.DESCENDING), ], background=True) mongo.collections['queue'].ensure_index('runner_id', background=True) mongo.collections['queue'].ensure_index('ttl_timestamp', background=True) mongo.collections['task'].ensure_index('type', background=True, unique=True) mongo.collections['task'].ensure_index('ttl_timestamp', background=True) mongo.collections['log_entries'].ensure_index([ ('timestamp', pymongo.DESCENDING), ], background=True) mongo.collections['messages'].ensure_index('channel', background=True) mongo.collections['administrators'].ensure_index('username', background=True, unique=True) mongo.collections['users'].ensure_index('resource_id', background=True) mongo.collections['users'].ensure_index([ ('type', pymongo.ASCENDING), ('org_id', pymongo.ASCENDING), ], background=True) mongo.collections['users'].ensure_index([ ('org_id', pymongo.ASCENDING), ('name', pymongo.ASCENDING), ], background=True) mongo.collections['users_key_link'].ensure_index('key_id', background=True) mongo.collections['users_key_link'].ensure_index('short_id', background=True, unique=True) mongo.collections['organizations'].ensure_index('type', background=True) mongo.collections['hosts'].ensure_index('name', background=True) mongo.collections['hosts_usage'].ensure_index([ ('host_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) mongo.collections['servers'].ensure_index('name', background=True) mongo.collections['servers'].ensure_index('ping_timestamp', background=True) mongo.collections['servers_output'].ensure_index([ ('server_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) mongo.collections['servers_output_link'].ensure_index([ ('server_id', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) mongo.collections['servers_bandwidth'].ensure_index([ ('server_id', pymongo.ASCENDING), ('period', pymongo.ASCENDING), ('timestamp', pymongo.ASCENDING), ], background=True) mongo.collections['servers_ip_pool'].ensure_index([ ('server_id', pymongo.ASCENDING), ('user_id', pymongo.ASCENDING), ], background=True) mongo.collections['servers_ip_pool'].ensure_index('user_id', background=True) mongo.collections['dh_params'].ensure_index('dh_param_bits', background=True) mongo.collections['auth_nonces'].ensure_index([ ('token', pymongo.ASCENDING), ('nonce', pymongo.ASCENDING), ], background=True, unique=True) mongo.collections['users_key_link'].ensure_index('timestamp', background=True, expireAfterSeconds=settings.app.key_link_timeout) mongo.collections['auth_sessions'].ensure_index('timestamp', background=True, expireAfterSeconds=settings.app.session_timeout) mongo.collections['auth_nonces'].ensure_index('timestamp', background=True, expireAfterSeconds=settings.app.auth_time_window * 2.1) mongo.collections['auth_limiter'].ensure_index('timestamp', background=True, expireAfterSeconds=settings.app.auth_limiter_ttl) mongo.collections['otp'].ensure_index('timestamp', background=True, expireAfterSeconds=120) mongo.collections['otp_cache'].ensure_index('timestamp', background=True, expireAfterSeconds=settings.user.otp_cache_ttl) if not auth.Administrator.collection.find_one(): auth.Administrator( username=DEFAULT_USERNAME, password=DEFAULT_PASSWORD, default=True, ).commit() secret_key = settings.app.cookie_secret if not secret_key: secret_key = re.sub(r'[\W_]+', '', base64.b64encode(os.urandom(128)))[:64] settings.app.cookie_secret = secret_key settings.commit() app.app.secret_key = secret_key.encode()