resweb_host='127.0.0.1',
resweb_port=7070,
persist_store_redis_host=DEVELOPMENT_REDIS_HOST,
persist_store_redis_port=DEVELOPMENT_REDIS_PORT,
memory_cache_redis_host=DEVELOPMENT_REDIS_HOST,
memory_cache_redis_port=DEVELOPMENT_REDIS_PORT)
ENV_DEVELOPMENT = {
'development': veil_env(name='development', hosts={}, servers={
'@': veil_server(
host_name='',
sequence_no=10,
supervisor_http_port=DEVELOPMENT_SUPERVISOR_HTTP_PORT,
programs=merge_multiple_settings(
redis_program('development', DEVELOPMENT_REDIS_HOST, DEVELOPMENT_REDIS_PORT),
__env__.tgcaem_postgresql_program(config),
__env__.resweb_program(config),
__env__.delayed_job_scheduler_program(config),
__env__.tgcaem_periodic_job_scheduler_program(config),
__env__.tgcaem_job_worker_program(
worker_name='development',
queue_names=[
'clean_up_captcha_images',
'clean_up_inline_static_files',
'send_transactional_email',
],
config=config, count=2),
__env__.teacher_website_programs(config),
__env__.operator_website_programs(config),
__env__.log_rotated_nginx_program(merge_multiple_settings(
__env__.resweb_nginx_server(config),
def tgcaem_public_servers(
host_name,
lan_range,
teacher_website_domain,
operator_website_domain,
is_production=True,
backup_mirror_domain=None,
backup_mirror_host_ip=None,
monitor_config=None,
):
worker_ip = "{}.20".format(lan_range)
db_ip = "{}.30".format(lan_range)
monitor_ip = "{}.98".format(lan_range)
postgresql_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "postgresql"}
json_event_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "json_event"}
config = __env__.env_config(
teacher_website_start_port=5000,
teacher_website_process_count=8,
teacher_website_domain=teacher_website_domain,
teacher_website_domain_port=80,
operator_website_start_port=5100,
operator_website_process_count=4,
operator_website_domain=operator_website_domain,
operator_website_domain_port=OPERATOR_WEBSITE_DOMAIN_PORT,
tgcaem_postgresql_version="9.4",
tgcaem_postgresql_host=db_ip,
tgcaem_postgresql_port=5432,
tgcaem_postgresql_enable_chinese_fts=True,
queue_type="redis",
queue_host=db_ip,
queue_port=6378,
resweb_domain="", # do not expose it to public
resweb_domain_port=0, # do not expose it to public
resweb_host=worker_ip,
resweb_port=7070,
persist_store_redis_host=db_ip,
persist_store_redis_port=6381,
memory_cache_redis_host=db_ip,
memory_cache_redis_port=6382,
)
tgcaem_postgresql_more_config = (
DictObject(
shared_buffers="512MB",
work_mem="32MB",
effective_io_concurrency=3,
checkpoint_segments=64,
checkpoint_completion_target=0.9,
effective_cache_size="256MB",
)
if is_production
else DictObject()
)
teacher_website_authority = (
config.teacher_website_domain
if 80 == config.teacher_website_domain_port
else "{}:{}".format(config.teacher_website_domain, config.teacher_website_domain_port)
)
config.redis_servers = [
(config.queue_host, config.queue_port),
(config.persist_store_redis_host, config.persist_store_redis_port),
(config.memory_cache_redis_host, config.memory_cache_redis_port),
]
return (
["@guard", "web", "worker", "db", "@monitor"],
{
"@guard": veil_server(
host_name=host_name,
sequence_no=99,
programs=guard_program("7 4 * * *"),
backup_mirror=None,
ssh_port=22,
),
"@monitor": veil_server(
host_name=host_name,
sequence_no=98,
programs=monitor_programs(
DictObject(
{
"log_buffer_redis_host": monitor_ip,
"log_buffer_redis_port": 5140,
"elasticsearch_host": monitor_ip,
"elasticsearch_port": 9200,
"elasticsearch_transport_port": 9300,
"es_heap_size": monitor_config.es_heap_size,
"ls_heap_size": monitor_config.ls_heap_size,
}
)
),
memory_limit=monitor_config.monitor_memory_limit,
cpu_share=monitor_config.monitor_cpu_share,
ssh_port=22,
),
"web": veil_server(
host_name=host_name,
sequence_no=10,
mount_buckets_dir=True,
programs=merge_multiple_settings(
__env__.teacher_website_programs(config),
__env__.operator_website_programs(config),
__env__.log_rotated_nginx_program(
merge_multiple_settings(
__env__.teacher_website_nginx_server(config),
__env__.operator_website_nginx_server(
config,
keepalive_timeout="120",
ssl=True,
ssl_session_timeout="15m",
ssl_certificate="/etc/ssl/certs/op.tgcaem.org.crt",
ssl_certificate_key="/etc/ssl/private/op.tgcaem.org.key",
),
nginx_server(
"_",
config.teacher_website_domain_port,
locations={},
rewrite="^ http://{}/".format(teacher_website_authority),
default_server=True,
),
),
enable_compression=True,
worker_process_count="auto" if is_production else 2,
worker_priority=-5,
worker_rlimit_nofile=130000,
worker_connections=2048,
server_names_hash_bucket_size=64,
),
log_shipper_program(
{
VEIL_LOG_DIR
/ "nginx"
/ "{}-access.log".format(teacher_website_domain): json_event_log_collector_config,
VEIL_LOG_DIR
/ "nginx"
/ "{}-access.log".format(operator_website_domain): json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado1-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado2-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado3-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado4-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado5-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado6-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado7-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "teacher_tornado8-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "operator_tornado1-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "operator_tornado2-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "operator_tornado3-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "operator_tornado4-stderr.log": json_event_log_collector_config,
}
),
),
ssh_port=22,
),
"worker": veil_server(
host_name=host_name,
sequence_no=20,
mount_buckets_dir=True,
programs=merge_multiple_settings(
__env__.resweb_program(config),
__env__.delayed_job_scheduler_program(config),
__env__.tgcaem_periodic_job_scheduler_program(config),
__env__.tgcaem_routines_job_worker_program(config, 2),
__env__.transactional_email_worker_program(config, 2),
log_shipper_program(
{
VEIL_LOG_DIR / "tgcaem_routines_worker1-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "tgcaem_routines_worker2-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "transactional_email_worker1-stderr.log": json_event_log_collector_config,
VEIL_LOG_DIR / "transactional_email_worker2-stderr.log": json_event_log_collector_config,
}
),
),
ssh_port=22,
),
"db": veil_server(
host_name=host_name,
sequence_no=30,
mount_data_dir=True,
programs=merge_multiple_settings(
__env__.persist_store_redis_program(config),
__env__.memory_cache_redis_program(config),
__env__.queue_program(config),
__env__.tgcaem_postgresql_program(config, tgcaem_postgresql_more_config),
log_shipper_program(
{VEIL_LOG_DIR / "tgcaem-postgresql" / "postgresql.csv": postgresql_log_collector_config}
),
),
resources=[
(
"veil.backend.database.client.database_client_resource",
dict(purpose="tgcaem", config=__env__.tgcaem_config(config).tgcaem_database_client),
)
],
ssh_port=22,
),
},
config,
)