def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_host=None, supervisor_http_port=None, name_servers=None, backup_mirror=None, mount_var_dir=False, mount_editorial_dir=False, mount_buckets_dir=False, mount_data_dir=False, mount_barman_dir=False, memory_limit=None, cpu_share=None, cpus=None): assert not mount_var_dir or not mount_editorial_dir and not mount_buckets_dir and not mount_data_dir and not mount_barman_dir from veil.model.collection import objectify if backup_mirror: backup_mirror = objectify(backup_mirror) backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port) return objectify({ 'host_name': host_name, 'sequence_no': sequence_no, 'programs': programs, 'resources': resources, 'supervisor_http_host': supervisor_http_host, 'supervisor_http_port': supervisor_http_port, 'name_servers': name_servers or DEFAULT_DNS_SERVERS, 'backup_mirror': backup_mirror, 'mount_var_dir': mount_var_dir, 'mount_editorial_dir': mount_editorial_dir, 'mount_buckets_dir': mount_buckets_dir, 'mount_data_dir': mount_data_dir, 'mount_barman_dir': mount_barman_dir, 'memory_limit': memory_limit, 'cpu_share': cpu_share, 'cpus': cpus })
def get_delivery_status(shipper_code, shipping_code, sleep_at_start=0): if 'youzhengguonei' == shipper_code: # kuaidi100 API does not support China Post yet return {} params = {'id': kuaidi100_client_config().api_id, 'com': shipper_code, 'nu': shipping_code, 'muti': '1', 'order': 'desc'} # sleep_at_start: avoid IP blocking due to too frequent queries if sleep_at_start > 0: sleep(sleep_at_start) try: response = requests.get(API_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9), max_retries=Retry(total=3, backoff_factor=0.5)) response.raise_for_status() except Exception: LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params}) else: result = objectify(response.json()) if result.status == STATUS_QUERY_SUCCESS: LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', {'result': result, 'url': response.url}) return result elif result.status == STATUS_QUERY_ERROR: LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', {'result': result, 'url': response.url}) elif result.status == STATUS_NO_INFO_YET: if all(keyword in result.message for keyword in ('IP地址', '禁止')): raise IPBlockedException('IP blocked by kuaidi100: {}'.format(result)) else: LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s', {'result': result, 'url': response.url}) return {}
def veil_env(server_hosts, servers, deployment_memo=None): from veil.model.collection import objectify return objectify({ 'server_hosts': server_hosts, 'servers': servers, 'deployment_memo': deployment_memo })
def get_delivery_provider(text): params = {'text': text} try: response = requests.get(AUTO_COM_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9), max_retries=Retry(total=3, backoff_factor=0.5)) response.raise_for_status() except Exception: LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params}) else: result = objectify(response.json()) return result
def veil_server(hosted_on, sequence_no, programs, deploys_via=None, resources=(), supervisor_http_port=None, dns='8.8.8.8'): from veil.model.collection import objectify return objectify({ 'hosted_on': hosted_on, 'sequence_no': sequence_no, 'programs': programs, 'deploys_via': deploys_via, 'resources': resources, 'supervisor_http_port': supervisor_http_port, 'dns': dns })
def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_port=None, name_servers=None, backup_mirror=None, mount_editorial_dir=False, mount_buckets_dir=False, mount_data_dir=False, memory_limit=None, cpu_share=None, cpus=None, ssh_port=None): from veil.model.collection import objectify if backup_mirror: backup_mirror = objectify(backup_mirror) backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port) return objectify({ 'host_name': host_name, 'sequence_no': sequence_no, 'programs': programs, 'resources': resources, 'supervisor_http_port': supervisor_http_port, 'name_servers': name_servers or ['119.29.29.29', '182.254.116.116', '8.8.8.8', '8.8.4.4'], 'backup_mirror': backup_mirror, 'mount_editorial_dir': mount_editorial_dir, 'mount_buckets_dir': mount_buckets_dir, 'mount_data_dir': mount_data_dir, 'memory_limit': memory_limit, 'cpu_share': cpu_share, 'cpus': cpus, 'ssh_port': ssh_port })
def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_host=None, supervisor_http_port=None, name_servers=None, backup_mirror=None, mount_editorial_dir=False, mount_buckets_dir=False, mount_data_dir=False, memory_limit=None, cpu_share=None, cpus=None, ssh_port=None): from veil.model.collection import objectify if backup_mirror: backup_mirror = objectify(backup_mirror) backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port) return objectify({ 'host_name': host_name, 'sequence_no': sequence_no, 'programs': programs, 'resources': resources, 'supervisor_http_host': supervisor_http_host, 'supervisor_http_port': supervisor_http_port, 'name_servers': name_servers or DEFAULT_DNS_SERVERS, 'backup_mirror': backup_mirror, 'mount_editorial_dir': mount_editorial_dir, 'mount_buckets_dir': mount_buckets_dir, 'mount_data_dir': mount_data_dir, 'memory_limit': memory_limit, 'cpu_share': cpu_share, 'cpus': cpus, 'ssh_port': ssh_port })
def veil_host(internal_ip, external_ip, ssh_port=22, ssh_user='******', lan_range='10.0.3', lan_interface='lxcbr0', mac_prefix='00:16:3e:73:bb', resources=()): from veil.model.collection import objectify return objectify({ 'internal_ip': internal_ip, 'external_ip': external_ip, 'ssh_port': ssh_port, 'ssh_user': ssh_user, 'lan_range': lan_range, 'lan_interface': lan_interface, 'mac_prefix': mac_prefix, 'resources': resources })
def veil_host(lan_range, lan_interface, mac_prefix, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(), timezone='Asia/Shanghai'): from veil.model.collection import objectify internal_ip = '{}.1'.format(lan_range) return objectify({ 'timezone': timezone, 'lan_range': lan_range, 'lan_interface': lan_interface, 'mac_prefix': mac_prefix, 'internal_ip': internal_ip, 'external_ip': external_ip, 'ssh_port': ssh_port, 'ssh_user': ssh_user, 'ssh_user_group': ssh_user, 'sshd_config': sshd_config, 'iptables_rule_resources': iptables_rule_resources, 'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port) })
def veil_host(lan_range, lan_interface, mac_prefix, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(), timezone='Asia/Shanghai', external_service_ports=()): assert 'PermitRootLogin no' not in sshd_config, 'guard needs login host as root with certificate' from veil.model.collection import objectify internal_ip = '{}.1'.format(lan_range) return objectify({ 'timezone': timezone, 'lan_range': lan_range, 'lan_interface': lan_interface, 'mac_prefix': mac_prefix, 'internal_ip': internal_ip, 'external_ip': external_ip, 'ssh_port': ssh_port, 'ssh_user': ssh_user, 'ssh_user_group': ssh_user, 'sshd_config': sshd_config, 'iptables_rule_resources': iptables_rule_resources, 'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port), 'external_service_ports': external_service_ports })
def veil_host(lan_range, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(), timezone=None, external_service_ports=(), lxd_port=8443): if sshd_config and 'PasswordAuthentication no' not in sshd_config: raise AssertionError('password authentication should not be allowed on host') if sshd_config and 'PermitRootLogin no' not in sshd_config: raise AssertionError('root login should not be allowed on host') from veil.model.collection import objectify internal_ip = '{}.1'.format(lan_range) return objectify({ 'timezone': timezone or get_application_timezone(), 'lan_range': lan_range, 'internal_ip': internal_ip, 'external_ip': external_ip, 'ssh_port': ssh_port, 'ssh_user': ssh_user, 'ssh_user_group': ssh_user, 'sshd_config': sshd_config, 'iptables_rule_resources': iptables_rule_resources, 'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port), 'external_service_ports': external_service_ports, 'lxd_port': lxd_port, 'lxd_endpoint': 'https://{}:{}'.format(internal_ip, lxd_port) })
def get_delivery_status_by_kuaidi100(shipper_code, shipping_code, sleep_at_start=0): """ @param shipper_code: kuaidi100 shipping provider code @param shipping_code: trace code @param sleep_at_start: back log time @return: empty object if got error, {signed:=BOOLEAN, rejected:=BOOLEAN, traces:=[{text:=STRING}]} """ params = {'id': kuaidi100_client_config().api_id, 'com': shipper_code, 'nu': shipping_code, 'muti': '1', 'order': 'desc'} # sleep_at_start: avoid IP blocking due to too frequent queries if sleep_at_start > 0: sleep(sleep_at_start) try: response = requests.get(API_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9), max_retries=Retry(total=3, backoff_factor=0.5)) response.raise_for_status() except Exception: LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params}) else: result = objectify(response.json()) if result.status == STATUS_QUERY_SUCCESS: LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', {'result': result, 'url': response.url}) ret = DictObject(signed=result.state == DELIVERY_STATE_SIGNED, rejected=result.state == DELIVERY_STATE_REJECTED) ret.traces = [DictObject(text=trace.context) for trace in result.data] if result.data[0].time: ret.delivered_at = convert_datetime_to_utc_timezone(datetime.strptime(result.data[0].time, '%Y-%m-%d %H:%M:%S')) else: ret.delivered_at = get_current_time() return ret elif result.status == STATUS_QUERY_ERROR: LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', {'result': result, 'url': response.url}) elif result.status == STATUS_NO_INFO_YET: if all(keyword in result.message for keyword in ('IP地址', '禁止')): raise IPBlockedException('IP blocked by kuaidi100: {}'.format(result)) else: LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s', {'result': result, 'url': response.url}) return {}
def get_delivery_status_by_kuaidi100(shipper_code, shipping_code, sleep_at_start=0): """ @param shipper_code: kuaidi100 shipping provider code @param shipping_code: trace code @param sleep_at_start: back log time @return: empty object if got error, {signed:=BOOLEAN, rejected:=BOOLEAN, traces:=[{text:=STRING}]} """ params = { 'id': kuaidi100_client_config().api_id, 'com': shipper_code, 'nu': shipping_code, 'muti': '1', 'order': 'desc' } # sleep_at_start: avoid IP blocking due to too frequent queries if sleep_at_start > 0: sleep(sleep_at_start) try: response = requests.get(API_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9), max_retries=Retry(total=3, backoff_factor=0.5)) response.raise_for_status() except Exception: LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params}) else: result = objectify(response.json()) if result.status == STATUS_QUERY_SUCCESS: LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', { 'result': result, 'url': response.url }) ret = DictObject(signed=result.state == DELIVERY_STATE_SIGNED, rejected=result.state == DELIVERY_STATE_REJECTED) ret.traces = [ DictObject(text=trace.context) for trace in result.data ] if result.data[0].time: ret.delivered_at = convert_datetime_to_utc_timezone( datetime.strptime(result.data[0].time, '%Y-%m-%d %H:%M:%S')) else: ret.delivered_at = get_current_time() return ret elif result.status == STATUS_QUERY_ERROR: LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', { 'result': result, 'url': response.url }) elif result.status == STATUS_NO_INFO_YET: if all(keyword in result.message for keyword in ('IP地址', '禁止')): raise IPBlockedException( 'IP blocked by kuaidi100: {}'.format(result)) else: LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s', { 'result': result, 'url': response.url }) return {}
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None): server_names = servers.keys() if sorted_server_names: assert set(sorted_server_names) == set(server_names), \ 'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names) assert 'monitor' not in sorted_server_names or 'monitor' == sorted_server_names[-1], \ 'ENV {}: monitor should be the last one in {}'.format(name, sorted_server_names) else: sorted_server_names = sorted(server_names) from veil.model.collection import objectify assert NAME_PATTERN.match(name) is not None, 'invalid characters in veil environment name: {}'.format(name) env = objectify({ 'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names, 'apt_url': apt_url, 'pypi_index_host': urlparse(pypi_index_url).hostname, 'pypi_index_url': pypi_index_url, 'deployment_memo': deployment_memo, 'config': config or {} }) env.VEIL_ENV = VeilEnv(env.name) env.env_dir = OPT_DIR / env.VEIL_ENV.base_name env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app' env.server_list = [] for server_name, server in env.servers.items(): assert NAME_PATTERN.match(server_name) is not None, 'invalid characters in veil server name: {}'.format(server_name) server.VEIL_ENV = env.VEIL_ENV server.name = server_name server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name) server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0 server.apt_url = env.apt_url server.pypi_index_host = env.pypi_index_host server.pypi_index_url = env.pypi_index_url server.veil_home = env.veil_home server.code_dir = server.veil_home.parent server.veil_framework_home = server.code_dir / 'veil' server.container_name = '{}-{}'.format(server.VEIL_ENV.name, server.name) server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name) server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path) server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name) server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name) server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name) server.host = None env.server_list.append(server) env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) for host_name, host in env.hosts.items(): assert NAME_PATTERN.match(host_name) is not None, 'invalid characters in veil host name: {}'.format(host_name) host.VEIL_ENV = env.VEIL_ENV host.name = host_name # host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name)) # 生产环境部署到多个host,staging只有一台host,用一台host模拟多台host时,在staging的host name加--1,--2,--3表示多台机器 host.base_name = host.name.rsplit('--', 1)[0] host.apt_url = env.apt_url host.pypi_index_host = env.pypi_index_host host.pypi_index_url = env.pypi_index_url host.ssh_user_home = as_path('/home') / host.ssh_user host.opt_dir = OPT_DIR host.share_dir = SHARE_DIR host.env_dir = env.env_dir host.etc_dir = host.env_dir / 'etc' host.log_dir = host.env_dir / 'log' host.var_dir = host.env_dir / 'var' host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial' host.buckets_dir = host.var_dir / 'buckets' host.bucket_log_dir = host.buckets_dir / 'log' host.bucket_inline_static_files_dir = host.buckets_dir / 'inline-static-files' host.bucket_captcha_image_dir = host.buckets_dir / 'captcha-image' host.bucket_uploaded_files_dir = host.buckets_dir / 'uploaded-files' host.data_dir = host.var_dir / 'data' host.veil_home = env.veil_home host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name) host.code_dir = host.veil_home.parent host.veil_framework_home = host.code_dir / 'veil' host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name) host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path) host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name) host.initialized_tag_link = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.base_name) host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name) host.with_user_editor = False host.server_list = [] for server_name, server in env.servers.items(): if host.name != server.host_name: continue server.host = host server.host_base_name = host.base_name server.ssh_user = host.ssh_user server.ssh_port = server.ssh_port or host.ssh_port server.ssh_user_group = host.ssh_user_group server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no) server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port) if server.backup_mirror: assert server.backup_mirror.host_ip.rsplit('.', 1)[0] != host.lan_range, \ 'ENV {}: local backup mirror does not make sense, please use remote mirror (on-site must, off-site optional)'.format(env.name) server.etc_dir = host.etc_dir / server.name server.log_dir = host.log_dir / server.name server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None server.data_dir = host.data_dir if server.mount_data_dir else None host.with_user_editor = host.with_user_editor or server.mount_editorial_dir host.server_list.append(server) host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) if env.hosts: assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name) assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name) assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in env.hosts.values()), \ 'ENV {}: found sequence no conflict among servers on one host'.format(env.name) assert all( server.name not in ('guard', 'monitor') or not server.mount_editorial_dir and not server.mount_buckets_dir and not server.mount_data_dir for server in env.servers.values()), 'ENV {}: found guard or monitor with editorial/buckets/data mount'.format(env.name) # break cyclic reference between host and server to get freeze_dict_object out of complain for server in env.servers.values(): del server.host return env
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None): """ Veil env. consists of multiple veil servers deployed on one or more veil hosts, every veil server is running as an LXD container. Recommended deployment of a Veil Env. consists of two hosts: Host #1: runs veil servers such as guard, web, db, worker Host #2: runs veil servers such as guard, barman, monitor If only one host available, it is okay to deploy the whole veil env. on one host. Server guard: 1. Periodically: makes host backup (except PostgreSQL) and synchronizes to backup mirror 2. Near-real-time: synchronizes latest updates for buckets and redis aof files to backup-mirror Every host requiring bucket&redis backup should run a guard server. At most one guard server is allowed for hosts with the same base name. Backup mirror is a special host configured on every guard server to mirror the backups on another host. Every guard server should have one backup mirror. At most one backup mirror is allowed in one env. At most one barman server is available for PostgreSQL server backup (PITR with streaming replication). At most one monitor server is allowed in one env. """ server_names = servers.keys() if sorted_server_names: assert set(sorted_server_names) == set(server_names), \ 'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names) assert 'monitor' not in sorted_server_names or 'monitor' == sorted_server_names[-1], \ 'ENV {}: monitor should be the last one in {}'.format(name, sorted_server_names) else: sorted_server_names = sorted(server_names) from veil.model.collection import objectify assert NAME_PATTERN.match(name) is not None, 'invalid characters in veil environment name: {}'.format(name) env = objectify({ 'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names, 'apt_url': apt_url, 'pypi_index_url': pypi_index_url, 'deployment_memo': deployment_memo, 'config': config or {} }) env.VEIL_ENV = VeilEnv(env.name) env.env_dir = OPT_DIR / env.VEIL_ENV.base_name env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app' env.server_list = [] env.backup_mirror = next((server.backup_mirror for server in env.servers.values() if server.backup_mirror), None) for server_name, server in env.servers.items(): assert NAME_PATTERN.match(server_name) is not None, 'invalid characters in veil server name: {}'.format(server_name) server.VEIL_ENV = env.VEIL_ENV server.is_guard = is_guard_server(server_name) server.is_barman = is_barman_server(server_name) server.is_monitor = is_monitor_server(server_name) server.name = server_name server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name) server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0 server.apt_url = env.apt_url server.pypi_index_url = env.pypi_index_url server.veil_home = env.veil_home server.code_dir = server.veil_home.parent server.veil_framework_home = server.code_dir / 'veil' server.container_name = '{}---{}'.format(server.VEIL_ENV.name, server.name) server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name) server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path) server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name) server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name) server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name) server.host = None env.server_list.append(server) env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) for host_name, host in env.hosts.items(): assert NAME_PATTERN.match(host_name) is not None, 'invalid characters in veil host name: {}'.format(host_name) host.VEIL_ENV = env.VEIL_ENV host.name = host_name # host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name)) # 生产环境部署到多个host,staging只有一台host,用一台host模拟多台host时,在staging的host name加--1,--2,--3表示多台机器 host.base_name = host.name.rsplit('--', 1)[0] host.apt_url = env.apt_url host.pypi_index_url = env.pypi_index_url host.ssh_user_home = as_path('/home') / host.ssh_user host.share_dir = SHARE_DIR host.env_dir = env.env_dir host.etc_dir = host.env_dir / 'etc' host.log_dir = host.env_dir / 'log' host.var_dir = host.env_dir / 'var' host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial' host.buckets_dir = host.var_dir / 'buckets' host.bucket_log_dir = host.buckets_dir / 'log' host.data_dir = host.var_dir / 'data' host.barman_dir = host.var_dir / 'barman' host.veil_home = env.veil_home host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name) host.code_dir = host.veil_home.parent host.veil_framework_home = host.code_dir / 'veil' host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name) host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path) host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name) host.initialized_tag_link = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.base_name) host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name) host.with_user_editor = False host.server_list = [] for server_name, server in env.servers.items(): if host.name != server.host_name: continue server.host = host server.lxd_endpoint = host.lxd_endpoint server.host_base_name = host.base_name server.ssh_user = host.ssh_user server.ssh_port = host.ssh_port server.ssh_user_group = host.ssh_user_group server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no) server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port) server.env_dir = host.env_dir server.etc_dir = host.etc_dir / server.name server.log_dir = host.log_dir / server.name server.var_dir = host.var_dir if server.mount_var_dir else None server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None server.data_dir = host.data_dir if server.mount_data_dir else None server.barman_dir = host.barman_dir if server.mount_barman_dir else None host.with_user_editor = host.with_user_editor or server.mount_editorial_dir host.server_list.append(server) host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) if env.hosts: assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name) assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name) assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in env.hosts.values()), 'ENV {}: found sequence no conflict among servers on one host'.format(env.name) assert all(len([server for server in servers if server.is_guard]) <= 1 for _, servers in itertools.groupby(sorted(env.servers.values(), key=operator.attrgetter('host_base_name')), operator.attrgetter('host_base_name')) ), 'ENV {}: found more than one guard on one host'.format(env.name) assert len([server for server in env.servers.values() if server.is_barman]) <= 1, 'ENV {}: found more than one barman'.format(env.name) assert len([server for server in env.servers.values() if server.is_monitor]) <= 1, 'ENV {}: found more than one monitor'.format(env.name) assert all(not server.is_guard or server.var_dir for server in env.servers.values()), 'ENV {}: found guard without var mount'.format(env.name) assert all(not server.is_barman or server.barman_dir for server in env.servers.values()), 'ENV {}: found barman without barman mount'.format(env.name) assert all(server.is_guard or not server.var_dir for server in env.servers.values()), 'ENV {}: found non-guard servers with var mount'.format(env.name) assert all( not server.is_monitor or not server.var_dir and not server.editorial_dir and not server.buckets_dir and not server.data_dir and not server.barman_dir for server in env.servers.values()), 'ENV {}: found monitor with var/editorial/buckets/data mount'.format(env.name) assert all( (server.is_guard or server.is_barman) and server.backup_mirror or not (server.is_guard or server.is_barman) and not server.backup_mirror for server in env.servers.values()), 'ENV {}: found guard/barman without backup mirror or non-guard/barman server with backup mirror'.format(env.name) assert all(not server.backup_mirror or env.backup_mirror == server.backup_mirror for server in env.servers.values()), \ 'ENV {}: found more than one backup mirror, at most one is allowed in one env.'.format(env.name) # break cyclic reference between host and server to get freeze_dict_object out of complain for server in env.servers.values(): del server.host return env
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None): server_names = servers.keys() if sorted_server_names: assert set(sorted_server_names) == set(server_names), \ 'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names) assert '@monitor' not in sorted_server_names or '@monitor' == sorted_server_names[-1], \ 'ENV {}: @monitor should be the last one in {}'.format(name, sorted_server_names) else: sorted_server_names = sorted(server_names) from veil.model.collection import objectify env = objectify({ 'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names, 'apt_url': apt_url, 'pypi_index_host': urlparse(pypi_index_url).hostname, 'pypi_index_url': pypi_index_url, 'deployment_memo': deployment_memo, 'config': config or {} }) env.VEIL_ENV = VeilEnv(env.name) env.env_dir = OPT_DIR / env.name env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app' env.server_list = [] for server_name, server in env.servers.items(): server.VEIL_ENV = env.VEIL_ENV server.name = server_name server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name) server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0 server.apt_url = env.apt_url server.pypi_index_host = env.pypi_index_host server.pypi_index_url = env.pypi_index_url server.veil_home = env.veil_home server.code_dir = server.veil_home.parent server.veil_framework_home = server.code_dir / 'veil' server.container_name = '{}-{}'.format(server.VEIL_ENV.name, server.name) server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name) server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path) server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name) server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name) server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name) server.host = None env.server_list.append(server) env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) for host_name, host in env.hosts.items(): host.VEIL_ENV = env.VEIL_ENV host.name = host_name # host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name)) host.base_name = host.name.split('/', 1)[0] # e.g. ljhost-90/1 => ljhost-90 host.apt_url = env.apt_url host.pypi_index_host = env.pypi_index_host host.pypi_index_url = env.pypi_index_url host.ssh_user_home = as_path('/home') / host.ssh_user host.opt_dir = OPT_DIR host.share_dir = SHARE_DIR host.env_dir = env.env_dir host.etc_dir = host.env_dir / 'etc' host.log_dir = host.env_dir / 'log' host.var_dir = host.env_dir / 'var' host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial' host.buckets_dir = host.var_dir / 'buckets' host.bucket_log_dir = host.buckets_dir / 'log' host.bucket_inline_static_files_dir = host.buckets_dir / 'inline-static-files' host.bucket_captcha_image_dir = host.buckets_dir / 'captcha-image' host.bucket_uploaded_files_dir = host.buckets_dir / 'uploaded-files' host.data_dir = host.var_dir / 'data' host.veil_home = env.veil_home host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name) host.code_dir = host.veil_home.parent host.veil_framework_home = host.code_dir / 'veil' host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name) host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path) host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name) host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name) host.with_user_editor = False host.server_list = [] for server_name, server in env.servers.items(): if host.name != server.host_name: continue server.host = host server.host_base_name = host.base_name server.ssh_user = host.ssh_user server.ssh_port = server.ssh_port or host.ssh_port server.ssh_user_group = host.ssh_user_group server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no) server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port) if server.backup_mirror: assert server.backup_mirror.host_ip.rsplit('.', 1)[0] != host.lan_range, \ 'ENV {}: local backup mirror does not make sense, please use remote mirror (on-site must, off-site optional)'.format(env.name) server.etc_dir = host.etc_dir / server.name server.log_dir = host.log_dir / server.name server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None server.data_dir = host.data_dir if server.mount_data_dir else None host.with_user_editor = host.with_user_editor or server.mount_editorial_dir host.server_list.append(server) host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name)) if env.hosts: assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name) assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name) assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in env.hosts.values()), \ 'ENV {}: found sequence no conflict among servers on one host'.format(env.name) assert all( server.name not in ('@guard', '@monitor') or not server.mount_editorial_dir and not server.mount_buckets_dir and not server.mount_data_dir for server in env.servers.values()), 'ENV {}: found @guard or @monitor with editorial/buckets/data mount'.format(env.name) # break cyclic reference between host and server to get freeze_dict_object out of complain for server in env.servers.values(): del server.host return env