resweb_host='127.0.0.1', resweb_port=7070, persist_store_redis_host=DEVELOPMENT_REDIS_HOST, persist_store_redis_port=DEVELOPMENT_REDIS_PORT, memory_cache_redis_host=DEVELOPMENT_REDIS_HOST, memory_cache_redis_port=DEVELOPMENT_REDIS_PORT) ENV_DEVELOPMENT = { 'development': veil_env(name='development', hosts={}, servers={ '@': veil_server( host_name='', sequence_no=10, supervisor_http_port=DEVELOPMENT_SUPERVISOR_HTTP_PORT, programs=merge_multiple_settings( redis_program('development', DEVELOPMENT_REDIS_HOST, DEVELOPMENT_REDIS_PORT), __env__.tgcaem_postgresql_program(config), __env__.resweb_program(config), __env__.delayed_job_scheduler_program(config), __env__.tgcaem_periodic_job_scheduler_program(config), __env__.tgcaem_job_worker_program( worker_name='development', queue_names=[ 'clean_up_captcha_images', 'clean_up_inline_static_files', 'send_transactional_email', ], config=config, count=2), __env__.teacher_website_programs(config), __env__.operator_website_programs(config), __env__.log_rotated_nginx_program(merge_multiple_settings( __env__.resweb_nginx_server(config),
def tgcaem_public_servers( host_name, lan_range, teacher_website_domain, operator_website_domain, is_production=True, backup_mirror_domain=None, backup_mirror_host_ip=None, monitor_config=None, ): worker_ip = "{}.20".format(lan_range) db_ip = "{}.30".format(lan_range) monitor_ip = "{}.98".format(lan_range) postgresql_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "postgresql"} json_event_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "json_event"} config = __env__.env_config( teacher_website_start_port=5000, teacher_website_process_count=8, teacher_website_domain=teacher_website_domain, teacher_website_domain_port=80, operator_website_start_port=5100, operator_website_process_count=4, operator_website_domain=operator_website_domain, operator_website_domain_port=OPERATOR_WEBSITE_DOMAIN_PORT, tgcaem_postgresql_version="9.4", tgcaem_postgresql_host=db_ip, tgcaem_postgresql_port=5432, tgcaem_postgresql_enable_chinese_fts=True, queue_type="redis", queue_host=db_ip, queue_port=6378, resweb_domain="", # do not expose it to public resweb_domain_port=0, # do not expose it to public resweb_host=worker_ip, resweb_port=7070, persist_store_redis_host=db_ip, persist_store_redis_port=6381, memory_cache_redis_host=db_ip, memory_cache_redis_port=6382, ) tgcaem_postgresql_more_config = ( DictObject( shared_buffers="512MB", work_mem="32MB", effective_io_concurrency=3, checkpoint_segments=64, checkpoint_completion_target=0.9, effective_cache_size="256MB", ) if is_production else DictObject() ) teacher_website_authority = ( config.teacher_website_domain if 80 == config.teacher_website_domain_port else "{}:{}".format(config.teacher_website_domain, config.teacher_website_domain_port) ) config.redis_servers = [ (config.queue_host, config.queue_port), (config.persist_store_redis_host, config.persist_store_redis_port), (config.memory_cache_redis_host, config.memory_cache_redis_port), ] return ( ["@guard", "web", "worker", "db", "@monitor"], { "@guard": veil_server( host_name=host_name, sequence_no=99, programs=guard_program("7 4 * * *"), backup_mirror=None, ssh_port=22, ), "@monitor": veil_server( host_name=host_name, sequence_no=98, programs=monitor_programs( DictObject( { "log_buffer_redis_host": monitor_ip, "log_buffer_redis_port": 5140, "elasticsearch_host": monitor_ip, "elasticsearch_port": 9200, "elasticsearch_transport_port": 9300, "es_heap_size": monitor_config.es_heap_size, "ls_heap_size": monitor_config.ls_heap_size, } ) ), memory_limit=monitor_config.monitor_memory_limit, cpu_share=monitor_config.monitor_cpu_share, ssh_port=22, ), "web": veil_server( host_name=host_name, sequence_no=10, mount_buckets_dir=True, programs=merge_multiple_settings( __env__.teacher_website_programs(config), __env__.operator_website_programs(config), __env__.log_rotated_nginx_program( merge_multiple_settings( __env__.teacher_website_nginx_server(config), __env__.operator_website_nginx_server( config, keepalive_timeout="120", ssl=True, ssl_session_timeout="15m", ssl_certificate="/etc/ssl/certs/op.tgcaem.org.crt", ssl_certificate_key="/etc/ssl/private/op.tgcaem.org.key", ), nginx_server( "_", config.teacher_website_domain_port, locations={}, rewrite="^ http://{}/".format(teacher_website_authority), default_server=True, ), ), enable_compression=True, worker_process_count="auto" if is_production else 2, worker_priority=-5, worker_rlimit_nofile=130000, worker_connections=2048, server_names_hash_bucket_size=64, ), log_shipper_program( { VEIL_LOG_DIR / "nginx" / "{}-access.log".format(teacher_website_domain): json_event_log_collector_config, VEIL_LOG_DIR / "nginx" / "{}-access.log".format(operator_website_domain): json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado1-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado2-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado3-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado4-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado5-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado6-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado7-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "teacher_tornado8-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "operator_tornado1-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "operator_tornado2-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "operator_tornado3-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "operator_tornado4-stderr.log": json_event_log_collector_config, } ), ), ssh_port=22, ), "worker": veil_server( host_name=host_name, sequence_no=20, mount_buckets_dir=True, programs=merge_multiple_settings( __env__.resweb_program(config), __env__.delayed_job_scheduler_program(config), __env__.tgcaem_periodic_job_scheduler_program(config), __env__.tgcaem_routines_job_worker_program(config, 2), __env__.transactional_email_worker_program(config, 2), log_shipper_program( { VEIL_LOG_DIR / "tgcaem_routines_worker1-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "tgcaem_routines_worker2-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "transactional_email_worker1-stderr.log": json_event_log_collector_config, VEIL_LOG_DIR / "transactional_email_worker2-stderr.log": json_event_log_collector_config, } ), ), ssh_port=22, ), "db": veil_server( host_name=host_name, sequence_no=30, mount_data_dir=True, programs=merge_multiple_settings( __env__.persist_store_redis_program(config), __env__.memory_cache_redis_program(config), __env__.queue_program(config), __env__.tgcaem_postgresql_program(config, tgcaem_postgresql_more_config), log_shipper_program( {VEIL_LOG_DIR / "tgcaem-postgresql" / "postgresql.csv": postgresql_log_collector_config} ), ), resources=[ ( "veil.backend.database.client.database_client_resource", dict(purpose="tgcaem", config=__env__.tgcaem_config(config).tgcaem_database_client), ) ], ssh_port=22, ), }, config, )