Esempio n. 1
0
    'development': veil_env(name='development', hosts={}, servers={
        '@': veil_server(
            host_name='',
            sequence_no=10,
            supervisor_http_port=DEVELOPMENT_SUPERVISOR_HTTP_PORT,
            programs=merge_multiple_settings(
                redis_program('development', DEVELOPMENT_REDIS_HOST, DEVELOPMENT_REDIS_PORT),
                __env__.tgcaem_postgresql_program(config),
                __env__.resweb_program(config),
                __env__.delayed_job_scheduler_program(config),
                __env__.tgcaem_periodic_job_scheduler_program(config),
                __env__.tgcaem_job_worker_program(
                    worker_name='development',
                    queue_names=[
                        'clean_up_captcha_images',
                        'clean_up_inline_static_files',
                        'send_transactional_email',
                    ],
                    config=config, count=2),
                __env__.teacher_website_programs(config),
                __env__.operator_website_programs(config),
                __env__.log_rotated_nginx_program(merge_multiple_settings(
                    __env__.resweb_nginx_server(config),
                    __env__.teacher_website_nginx_server(config),
                    __env__.operator_website_nginx_server(config),
                ))
            )
        )
    })
}
Esempio n. 2
0
    resweb_domain='',  # no resweb
    resweb_domain_port=0,
    resweb_host='',
    resweb_port=0)

ENV_TEST = {
    'test': veil_env(name='test', hosts={}, servers={
        '@': veil_server(
            host_name='',
            sequence_no=20,
            supervisor_http_port=TEST_SUPERVISOR_HTTP_PORT,
            programs=merge_multiple_settings(
                redis_program('test', TEST_REDIS_HOST, TEST_REDIS_PORT),
                __env__.vsee_postgresql_program(config),
                __env__.log_rotated_nginx_program(merge_multiple_settings(
                    __env__.person_website_nginx_server(config),
                ))
            ),
            resources=[application_resource(component_names=list_all_components(), config=merge_settings(
                __env__.vsee_config(config), {
                    'test_bucket': {
                        'type': 'filesystem',
                        'base_directory': '',
                        'base_url': ''
                    }
                }
            ))]
        )
    })
}
Esempio n. 3
0
             hosts={},
             servers={
                 '@':
                 veil_server(
                     host_name='',
                     sequence_no=10,
                     supervisor_http_port=DEVELOPMENT_SUPERVISOR_HTTP_PORT,
                     programs=merge_multiple_settings(
                         redis_program('development', DEVELOPMENT_REDIS_HOST,
                                       DEVELOPMENT_REDIS_PORT),
                         __env__.vsee_postgresql_program(config),
                         __env__.resweb_program(config),
                         __env__.delayed_job_scheduler_program(config),
                         __env__.vsee_periodic_job_scheduler_program(config),
                         __env__.vsee_job_worker_program(
                             worker_name='development',
                             queue_names=[
                                 'send_transactional_email',
                                 'clean_up_captcha_images',
                                 'clean_up_inline_static_files',
                             ],
                             config=config,
                             count=2), __env__.person_website_programs(config),
                         __env__.log_rotated_nginx_program(
                             merge_multiple_settings(
                                 __env__.resweb_nginx_server(config),
                                 __env__.person_website_nginx_server(config),
                             ))))
             })
}
Esempio n. 4
0
ENV_TEST = {
    'test':
    veil_env(
        name='test',
        hosts={},
        servers={
            'test-server':
            veil_server(
                host_name='',
                sequence_no=20,
                supervisor_http_port=TEST_SUPERVISOR_HTTP_PORT,
                programs=merge_multiple_settings(
                    redis_program('test', TEST_REDIS_HOST, TEST_REDIS_PORT),
                    __env__.cmcloud_postgresql_program(config),
                    __env__.log_rotated_nginx_program(
                        merge_multiple_settings(
                            __env__.teacher_website_nginx_server(config), ))),
                resources=[
                    application_resource(component_names=list_all_components(),
                                         config=merge_settings(
                                             __env__.cmcloud_config(config), {
                                                 'test_bucket': {
                                                     'type': 'filesystem',
                                                     'base_directory': '',
                                                     'base_url': ''
                                                 }
                                             }))
                ])
        })
}
Esempio n. 5
0
def tgcaem_public_servers(
    host_name,
    lan_range,
    teacher_website_domain,
    operator_website_domain,
    is_production=True,
    backup_mirror_domain=None,
    backup_mirror_host_ip=None,
    monitor_config=None,
):
    worker_ip = "{}.20".format(lan_range)
    db_ip = "{}.30".format(lan_range)
    monitor_ip = "{}.98".format(lan_range)

    postgresql_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "postgresql"}
    json_event_log_collector_config = {"host": monitor_ip, "port": LOG_REDIS_PORT, "key": "json_event"}
    config = __env__.env_config(
        teacher_website_start_port=5000,
        teacher_website_process_count=8,
        teacher_website_domain=teacher_website_domain,
        teacher_website_domain_port=80,
        operator_website_start_port=5100,
        operator_website_process_count=4,
        operator_website_domain=operator_website_domain,
        operator_website_domain_port=OPERATOR_WEBSITE_DOMAIN_PORT,
        tgcaem_postgresql_version="9.4",
        tgcaem_postgresql_host=db_ip,
        tgcaem_postgresql_port=5432,
        tgcaem_postgresql_enable_chinese_fts=True,
        queue_type="redis",
        queue_host=db_ip,
        queue_port=6378,
        resweb_domain="",  # do not expose it to public
        resweb_domain_port=0,  # do not expose it to public
        resweb_host=worker_ip,
        resweb_port=7070,
        persist_store_redis_host=db_ip,
        persist_store_redis_port=6381,
        memory_cache_redis_host=db_ip,
        memory_cache_redis_port=6382,
    )
    tgcaem_postgresql_more_config = (
        DictObject(
            shared_buffers="512MB",
            work_mem="32MB",
            effective_io_concurrency=3,
            checkpoint_segments=64,
            checkpoint_completion_target=0.9,
            effective_cache_size="256MB",
        )
        if is_production
        else DictObject()
    )
    teacher_website_authority = (
        config.teacher_website_domain
        if 80 == config.teacher_website_domain_port
        else "{}:{}".format(config.teacher_website_domain, config.teacher_website_domain_port)
    )

    config.redis_servers = [
        (config.queue_host, config.queue_port),
        (config.persist_store_redis_host, config.persist_store_redis_port),
        (config.memory_cache_redis_host, config.memory_cache_redis_port),
    ]

    return (
        ["@guard", "web", "worker", "db", "@monitor"],
        {
            "@guard": veil_server(
                host_name=host_name,
                sequence_no=99,
                programs=guard_program("7 4 * * *"),
                backup_mirror=None,
                ssh_port=22,
            ),
            "@monitor": veil_server(
                host_name=host_name,
                sequence_no=98,
                programs=monitor_programs(
                    DictObject(
                        {
                            "log_buffer_redis_host": monitor_ip,
                            "log_buffer_redis_port": 5140,
                            "elasticsearch_host": monitor_ip,
                            "elasticsearch_port": 9200,
                            "elasticsearch_transport_port": 9300,
                            "es_heap_size": monitor_config.es_heap_size,
                            "ls_heap_size": monitor_config.ls_heap_size,
                        }
                    )
                ),
                memory_limit=monitor_config.monitor_memory_limit,
                cpu_share=monitor_config.monitor_cpu_share,
                ssh_port=22,
            ),
            "web": veil_server(
                host_name=host_name,
                sequence_no=10,
                mount_buckets_dir=True,
                programs=merge_multiple_settings(
                    __env__.teacher_website_programs(config),
                    __env__.operator_website_programs(config),
                    __env__.log_rotated_nginx_program(
                        merge_multiple_settings(
                            __env__.teacher_website_nginx_server(config),
                            __env__.operator_website_nginx_server(
                                config,
                                keepalive_timeout="120",
                                ssl=True,
                                ssl_session_timeout="15m",
                                ssl_certificate="/etc/ssl/certs/op.tgcaem.org.crt",
                                ssl_certificate_key="/etc/ssl/private/op.tgcaem.org.key",
                            ),
                            nginx_server(
                                "_",
                                config.teacher_website_domain_port,
                                locations={},
                                rewrite="^ http://{}/".format(teacher_website_authority),
                                default_server=True,
                            ),
                        ),
                        enable_compression=True,
                        worker_process_count="auto" if is_production else 2,
                        worker_priority=-5,
                        worker_rlimit_nofile=130000,
                        worker_connections=2048,
                        server_names_hash_bucket_size=64,
                    ),
                    log_shipper_program(
                        {
                            VEIL_LOG_DIR
                            / "nginx"
                            / "{}-access.log".format(teacher_website_domain): json_event_log_collector_config,
                            VEIL_LOG_DIR
                            / "nginx"
                            / "{}-access.log".format(operator_website_domain): json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado1-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado2-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado3-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado4-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado5-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado6-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado7-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "teacher_tornado8-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "operator_tornado1-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "operator_tornado2-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "operator_tornado3-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "operator_tornado4-stderr.log": json_event_log_collector_config,
                        }
                    ),
                ),
                ssh_port=22,
            ),
            "worker": veil_server(
                host_name=host_name,
                sequence_no=20,
                mount_buckets_dir=True,
                programs=merge_multiple_settings(
                    __env__.resweb_program(config),
                    __env__.delayed_job_scheduler_program(config),
                    __env__.tgcaem_periodic_job_scheduler_program(config),
                    __env__.tgcaem_routines_job_worker_program(config, 2),
                    __env__.transactional_email_worker_program(config, 2),
                    log_shipper_program(
                        {
                            VEIL_LOG_DIR / "tgcaem_routines_worker1-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "tgcaem_routines_worker2-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "transactional_email_worker1-stderr.log": json_event_log_collector_config,
                            VEIL_LOG_DIR / "transactional_email_worker2-stderr.log": json_event_log_collector_config,
                        }
                    ),
                ),
                ssh_port=22,
            ),
            "db": veil_server(
                host_name=host_name,
                sequence_no=30,
                mount_data_dir=True,
                programs=merge_multiple_settings(
                    __env__.persist_store_redis_program(config),
                    __env__.memory_cache_redis_program(config),
                    __env__.queue_program(config),
                    __env__.tgcaem_postgresql_program(config, tgcaem_postgresql_more_config),
                    log_shipper_program(
                        {VEIL_LOG_DIR / "tgcaem-postgresql" / "postgresql.csv": postgresql_log_collector_config}
                    ),
                ),
                resources=[
                    (
                        "veil.backend.database.client.database_client_resource",
                        dict(purpose="tgcaem", config=__env__.tgcaem_config(config).tgcaem_database_client),
                    )
                ],
                ssh_port=22,
            ),
        },
        config,
    )