def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_host=None, supervisor_http_port=None,
name_servers=None, backup_mirror=None, mount_var_dir=False, mount_editorial_dir=False, mount_buckets_dir=False,
mount_data_dir=False, mount_barman_dir=False, memory_limit=None, cpu_share=None, cpus=None):
assert not mount_var_dir or not mount_editorial_dir and not mount_buckets_dir and not mount_data_dir and not mount_barman_dir
from veil.model.collection import objectify
if backup_mirror:
backup_mirror = objectify(backup_mirror)
backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port)
return objectify({
'host_name': host_name,
'sequence_no': sequence_no,
'programs': programs,
'resources': resources,
'supervisor_http_host': supervisor_http_host,
'supervisor_http_port': supervisor_http_port,
'name_servers': name_servers or DEFAULT_DNS_SERVERS,
'backup_mirror': backup_mirror,
'mount_var_dir': mount_var_dir,
'mount_editorial_dir': mount_editorial_dir,
'mount_buckets_dir': mount_buckets_dir,
'mount_data_dir': mount_data_dir,
'mount_barman_dir': mount_barman_dir,
'memory_limit': memory_limit,
'cpu_share': cpu_share,
'cpus': cpus
})
def get_delivery_status(shipper_code, shipping_code, sleep_at_start=0):
if 'youzhengguonei' == shipper_code: # kuaidi100 API does not support China Post yet
return {}
params = {'id': kuaidi100_client_config().api_id, 'com': shipper_code, 'nu': shipping_code, 'muti': '1', 'order': 'desc'}
# sleep_at_start: avoid IP blocking due to too frequent queries
if sleep_at_start > 0:
sleep(sleep_at_start)
try:
response = requests.get(API_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9),
max_retries=Retry(total=3, backoff_factor=0.5))
response.raise_for_status()
except Exception:
LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params})
else:
result = objectify(response.json())
if result.status == STATUS_QUERY_SUCCESS:
LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', {'result': result, 'url': response.url})
return result
elif result.status == STATUS_QUERY_ERROR:
LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', {'result': result, 'url': response.url})
elif result.status == STATUS_NO_INFO_YET:
if all(keyword in result.message for keyword in ('IP地址', '禁止')):
raise IPBlockedException('IP blocked by kuaidi100: {}'.format(result))
else:
LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s', {'result': result, 'url': response.url})
return {}
def veil_env(server_hosts, servers, deployment_memo=None):
from veil.model.collection import objectify
return objectify({
'server_hosts': server_hosts,
'servers': servers,
'deployment_memo': deployment_memo
})
def get_delivery_provider(text):
params = {'text': text}
try:
response = requests.get(AUTO_COM_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9),
max_retries=Retry(total=3, backoff_factor=0.5))
response.raise_for_status()
except Exception:
LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params})
else:
result = objectify(response.json())
return result
def veil_server(hosted_on, sequence_no, programs, deploys_via=None, resources=(), supervisor_http_port=None,
dns='8.8.8.8'):
from veil.model.collection import objectify
return objectify({
'hosted_on': hosted_on,
'sequence_no': sequence_no,
'programs': programs,
'deploys_via': deploys_via,
'resources': resources,
'supervisor_http_port': supervisor_http_port,
'dns': dns
})
def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_port=None, name_servers=None, backup_mirror=None, mount_editorial_dir=False,
mount_buckets_dir=False, mount_data_dir=False, memory_limit=None, cpu_share=None, cpus=None, ssh_port=None):
from veil.model.collection import objectify
if backup_mirror:
backup_mirror = objectify(backup_mirror)
backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port)
return objectify({
'host_name': host_name,
'sequence_no': sequence_no,
'programs': programs,
'resources': resources,
'supervisor_http_port': supervisor_http_port,
'name_servers': name_servers or ['119.29.29.29', '182.254.116.116', '8.8.8.8', '8.8.4.4'],
'backup_mirror': backup_mirror,
'mount_editorial_dir': mount_editorial_dir,
'mount_buckets_dir': mount_buckets_dir,
'mount_data_dir': mount_data_dir,
'memory_limit': memory_limit,
'cpu_share': cpu_share,
'cpus': cpus,
'ssh_port': ssh_port
})
def get_delivery_provider(text):
params = {'text': text}
try:
response = requests.get(AUTO_COM_URL,
params=params,
headers={'Accept': 'application/json'},
timeout=(3.05, 9),
max_retries=Retry(total=3, backoff_factor=0.5))
response.raise_for_status()
except Exception:
LOGGER.exception('kuaidi100 query exception-thrown: %(params)s',
{'params': params})
else:
result = objectify(response.json())
return result
def veil_server(host_name, sequence_no, programs, resources=(), supervisor_http_host=None, supervisor_http_port=None,
name_servers=None, backup_mirror=None, mount_editorial_dir=False, mount_buckets_dir=False,
mount_data_dir=False, memory_limit=None, cpu_share=None, cpus=None, ssh_port=None):
from veil.model.collection import objectify
if backup_mirror:
backup_mirror = objectify(backup_mirror)
backup_mirror.deploys_via = '{}@{}:{}'.format(backup_mirror.ssh_user, backup_mirror.host_ip, backup_mirror.ssh_port)
return objectify({
'host_name': host_name,
'sequence_no': sequence_no,
'programs': programs,
'resources': resources,
'supervisor_http_host': supervisor_http_host,
'supervisor_http_port': supervisor_http_port,
'name_servers': name_servers or DEFAULT_DNS_SERVERS,
'backup_mirror': backup_mirror,
'mount_editorial_dir': mount_editorial_dir,
'mount_buckets_dir': mount_buckets_dir,
'mount_data_dir': mount_data_dir,
'memory_limit': memory_limit,
'cpu_share': cpu_share,
'cpus': cpus,
'ssh_port': ssh_port
})
def veil_host(internal_ip, external_ip, ssh_port=22, ssh_user='******',
lan_range='10.0.3', lan_interface='lxcbr0', mac_prefix='00:16:3e:73:bb',
resources=()):
from veil.model.collection import objectify
return objectify({
'internal_ip': internal_ip,
'external_ip': external_ip,
'ssh_port': ssh_port,
'ssh_user': ssh_user,
'lan_range': lan_range,
'lan_interface': lan_interface,
'mac_prefix': mac_prefix,
'resources': resources
})
def veil_host(lan_range, lan_interface, mac_prefix, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(),
timezone='Asia/Shanghai'):
from veil.model.collection import objectify
internal_ip = '{}.1'.format(lan_range)
return objectify({
'timezone': timezone,
'lan_range': lan_range,
'lan_interface': lan_interface,
'mac_prefix': mac_prefix,
'internal_ip': internal_ip,
'external_ip': external_ip,
'ssh_port': ssh_port,
'ssh_user': ssh_user,
'ssh_user_group': ssh_user,
'sshd_config': sshd_config,
'iptables_rule_resources': iptables_rule_resources,
'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port)
})
def veil_host(lan_range, lan_interface, mac_prefix, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(),
timezone='Asia/Shanghai', external_service_ports=()):
assert 'PermitRootLogin no' not in sshd_config, 'guard needs login host as root with certificate'
from veil.model.collection import objectify
internal_ip = '{}.1'.format(lan_range)
return objectify({
'timezone': timezone,
'lan_range': lan_range,
'lan_interface': lan_interface,
'mac_prefix': mac_prefix,
'internal_ip': internal_ip,
'external_ip': external_ip,
'ssh_port': ssh_port,
'ssh_user': ssh_user,
'ssh_user_group': ssh_user,
'sshd_config': sshd_config,
'iptables_rule_resources': iptables_rule_resources,
'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port),
'external_service_ports': external_service_ports
})
def veil_host(lan_range, external_ip, ssh_port=22, ssh_user='******', sshd_config=(), iptables_rule_resources=(), timezone=None, external_service_ports=(),
lxd_port=8443):
if sshd_config and 'PasswordAuthentication no' not in sshd_config:
raise AssertionError('password authentication should not be allowed on host')
if sshd_config and 'PermitRootLogin no' not in sshd_config:
raise AssertionError('root login should not be allowed on host')
from veil.model.collection import objectify
internal_ip = '{}.1'.format(lan_range)
return objectify({
'timezone': timezone or get_application_timezone(),
'lan_range': lan_range,
'internal_ip': internal_ip,
'external_ip': external_ip,
'ssh_port': ssh_port,
'ssh_user': ssh_user,
'ssh_user_group': ssh_user,
'sshd_config': sshd_config,
'iptables_rule_resources': iptables_rule_resources,
'deploys_via': '{}@{}:{}'.format(ssh_user, internal_ip, ssh_port),
'external_service_ports': external_service_ports,
'lxd_port': lxd_port,
'lxd_endpoint': 'https://{}:{}'.format(internal_ip, lxd_port)
})
def get_delivery_status_by_kuaidi100(shipper_code, shipping_code, sleep_at_start=0):
"""
@param shipper_code: kuaidi100 shipping provider code
@param shipping_code: trace code
@param sleep_at_start: back log time
@return: empty object if got error, {signed:=BOOLEAN, rejected:=BOOLEAN, traces:=[{text:=STRING}]}
"""
params = {'id': kuaidi100_client_config().api_id, 'com': shipper_code, 'nu': shipping_code, 'muti': '1', 'order': 'desc'}
# sleep_at_start: avoid IP blocking due to too frequent queries
if sleep_at_start > 0:
sleep(sleep_at_start)
try:
response = requests.get(API_URL, params=params, headers={'Accept': 'application/json'}, timeout=(3.05, 9),
max_retries=Retry(total=3, backoff_factor=0.5))
response.raise_for_status()
except Exception:
LOGGER.exception('kuaidi100 query exception-thrown: %(params)s', {'params': params})
else:
result = objectify(response.json())
if result.status == STATUS_QUERY_SUCCESS:
LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', {'result': result, 'url': response.url})
ret = DictObject(signed=result.state == DELIVERY_STATE_SIGNED, rejected=result.state == DELIVERY_STATE_REJECTED)
ret.traces = [DictObject(text=trace.context) for trace in result.data]
if result.data[0].time:
ret.delivered_at = convert_datetime_to_utc_timezone(datetime.strptime(result.data[0].time, '%Y-%m-%d %H:%M:%S'))
else:
ret.delivered_at = get_current_time()
return ret
elif result.status == STATUS_QUERY_ERROR:
LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', {'result': result, 'url': response.url})
elif result.status == STATUS_NO_INFO_YET:
if all(keyword in result.message for keyword in ('IP地址', '禁止')):
raise IPBlockedException('IP blocked by kuaidi100: {}'.format(result))
else:
LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s', {'result': result, 'url': response.url})
return {}
def get_delivery_status_by_kuaidi100(shipper_code,
shipping_code,
sleep_at_start=0):
"""
@param shipper_code: kuaidi100 shipping provider code
@param shipping_code: trace code
@param sleep_at_start: back log time
@return: empty object if got error, {signed:=BOOLEAN, rejected:=BOOLEAN, traces:=[{text:=STRING}]}
"""
params = {
'id': kuaidi100_client_config().api_id,
'com': shipper_code,
'nu': shipping_code,
'muti': '1',
'order': 'desc'
}
# sleep_at_start: avoid IP blocking due to too frequent queries
if sleep_at_start > 0:
sleep(sleep_at_start)
try:
response = requests.get(API_URL,
params=params,
headers={'Accept': 'application/json'},
timeout=(3.05, 9),
max_retries=Retry(total=3, backoff_factor=0.5))
response.raise_for_status()
except Exception:
LOGGER.exception('kuaidi100 query exception-thrown: %(params)s',
{'params': params})
else:
result = objectify(response.json())
if result.status == STATUS_QUERY_SUCCESS:
LOGGER.debug('kuaidi100 query succeeded: %(result)s, %(url)s', {
'result': result,
'url': response.url
})
ret = DictObject(signed=result.state == DELIVERY_STATE_SIGNED,
rejected=result.state == DELIVERY_STATE_REJECTED)
ret.traces = [
DictObject(text=trace.context) for trace in result.data
]
if result.data[0].time:
ret.delivered_at = convert_datetime_to_utc_timezone(
datetime.strptime(result.data[0].time,
'%Y-%m-%d %H:%M:%S'))
else:
ret.delivered_at = get_current_time()
return ret
elif result.status == STATUS_QUERY_ERROR:
LOGGER.error('kuaidi100 query error: %(result)s, %(url)s', {
'result': result,
'url': response.url
})
elif result.status == STATUS_NO_INFO_YET:
if all(keyword in result.message for keyword in ('IP地址', '禁止')):
raise IPBlockedException(
'IP blocked by kuaidi100: {}'.format(result))
else:
LOGGER.info('kuaidi100 query no info yet: %(result)s, %(url)s',
{
'result': result,
'url': response.url
})
return {}
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None):
server_names = servers.keys()
if sorted_server_names:
assert set(sorted_server_names) == set(server_names), \
'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names)
assert 'monitor' not in sorted_server_names or 'monitor' == sorted_server_names[-1], \
'ENV {}: monitor should be the last one in {}'.format(name, sorted_server_names)
else:
sorted_server_names = sorted(server_names)
from veil.model.collection import objectify
assert NAME_PATTERN.match(name) is not None, 'invalid characters in veil environment name: {}'.format(name)
env = objectify({
'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names,
'apt_url': apt_url, 'pypi_index_host': urlparse(pypi_index_url).hostname, 'pypi_index_url': pypi_index_url,
'deployment_memo': deployment_memo, 'config': config or {}
})
env.VEIL_ENV = VeilEnv(env.name)
env.env_dir = OPT_DIR / env.VEIL_ENV.base_name
env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app'
env.server_list = []
for server_name, server in env.servers.items():
assert NAME_PATTERN.match(server_name) is not None, 'invalid characters in veil server name: {}'.format(server_name)
server.VEIL_ENV = env.VEIL_ENV
server.name = server_name
server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name)
server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0
server.apt_url = env.apt_url
server.pypi_index_host = env.pypi_index_host
server.pypi_index_url = env.pypi_index_url
server.veil_home = env.veil_home
server.code_dir = server.veil_home.parent
server.veil_framework_home = server.code_dir / 'veil'
server.container_name = '{}-{}'.format(server.VEIL_ENV.name, server.name)
server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name)
server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path)
server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name)
server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name)
server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name)
server.host = None
env.server_list.append(server)
env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
for host_name, host in env.hosts.items():
assert NAME_PATTERN.match(host_name) is not None, 'invalid characters in veil host name: {}'.format(host_name)
host.VEIL_ENV = env.VEIL_ENV
host.name = host_name
# host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name))
# 生产环境部署到多个host，staging只有一台host，用一台host模拟多台host时，在staging的host name加--1,--2,--3表示多台机器
host.base_name = host.name.rsplit('--', 1)[0]
host.apt_url = env.apt_url
host.pypi_index_host = env.pypi_index_host
host.pypi_index_url = env.pypi_index_url
host.ssh_user_home = as_path('/home') / host.ssh_user
host.opt_dir = OPT_DIR
host.share_dir = SHARE_DIR
host.env_dir = env.env_dir
host.etc_dir = host.env_dir / 'etc'
host.log_dir = host.env_dir / 'log'
host.var_dir = host.env_dir / 'var'
host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial'
host.buckets_dir = host.var_dir / 'buckets'
host.bucket_log_dir = host.buckets_dir / 'log'
host.bucket_inline_static_files_dir = host.buckets_dir / 'inline-static-files'
host.bucket_captcha_image_dir = host.buckets_dir / 'captcha-image'
host.bucket_uploaded_files_dir = host.buckets_dir / 'uploaded-files'
host.data_dir = host.var_dir / 'data'
host.veil_home = env.veil_home
host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name)
host.code_dir = host.veil_home.parent
host.veil_framework_home = host.code_dir / 'veil'
host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name)
host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path)
host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name)
host.initialized_tag_link = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.base_name)
host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name)
host.with_user_editor = False
host.server_list = []
for server_name, server in env.servers.items():
if host.name != server.host_name:
continue
server.host = host
server.host_base_name = host.base_name
server.ssh_user = host.ssh_user
server.ssh_port = server.ssh_port or host.ssh_port
server.ssh_user_group = host.ssh_user_group
server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no)
server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port)
if server.backup_mirror:
assert server.backup_mirror.host_ip.rsplit('.', 1)[0] != host.lan_range, \
'ENV {}: local backup mirror does not make sense, please use remote mirror (on-site must, off-site optional)'.format(env.name)
server.etc_dir = host.etc_dir / server.name
server.log_dir = host.log_dir / server.name
server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None
server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None
server.data_dir = host.data_dir if server.mount_data_dir else None
host.with_user_editor = host.with_user_editor or server.mount_editorial_dir
host.server_list.append(server)
host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
if env.hosts:
assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name)
assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name)
assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in env.hosts.values()), \
'ENV {}: found sequence no conflict among servers on one host'.format(env.name)
assert all(
server.name not in ('guard', 'monitor') or not server.mount_editorial_dir and not server.mount_buckets_dir and not server.mount_data_dir
for server in env.servers.values()), 'ENV {}: found guard or monitor with editorial/buckets/data mount'.format(env.name)
# break cyclic reference between host and server to get freeze_dict_object out of complain
for server in env.servers.values():
del server.host
return env
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None):
"""
Veil env. consists of multiple veil servers deployed on one or more veil hosts, every veil server is running as an LXD container.
Recommended deployment of a Veil Env. consists of two hosts:
Host #1: runs veil servers such as guard, web, db, worker
Host #2: runs veil servers such as guard, barman, monitor
If only one host available, it is okay to deploy the whole veil env. on one host.
Server guard:
1. Periodically: makes host backup (except PostgreSQL) and synchronizes to backup mirror
2. Near-real-time: synchronizes latest updates for buckets and redis aof files to backup-mirror
Every host requiring bucket&redis backup should run a guard server.
At most one guard server is allowed for hosts with the same base name.
Backup mirror is a special host configured on every guard server to mirror the backups on another host.
Every guard server should have one backup mirror.
At most one backup mirror is allowed in one env.
At most one barman server is available for PostgreSQL server backup (PITR with streaming replication).
At most one monitor server is allowed in one env.
"""
server_names = servers.keys()
if sorted_server_names:
assert set(sorted_server_names) == set(server_names), \
'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names)
assert 'monitor' not in sorted_server_names or 'monitor' == sorted_server_names[-1], \
'ENV {}: monitor should be the last one in {}'.format(name, sorted_server_names)
else:
sorted_server_names = sorted(server_names)
from veil.model.collection import objectify
assert NAME_PATTERN.match(name) is not None, 'invalid characters in veil environment name: {}'.format(name)
env = objectify({
'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names,
'apt_url': apt_url,
'pypi_index_url': pypi_index_url,
'deployment_memo': deployment_memo, 'config': config or {}
})
env.VEIL_ENV = VeilEnv(env.name)
env.env_dir = OPT_DIR / env.VEIL_ENV.base_name
env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app'
env.server_list = []
env.backup_mirror = next((server.backup_mirror for server in env.servers.values() if server.backup_mirror), None)
for server_name, server in env.servers.items():
assert NAME_PATTERN.match(server_name) is not None, 'invalid characters in veil server name: {}'.format(server_name)
server.VEIL_ENV = env.VEIL_ENV
server.is_guard = is_guard_server(server_name)
server.is_barman = is_barman_server(server_name)
server.is_monitor = is_monitor_server(server_name)
server.name = server_name
server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name)
server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0
server.apt_url = env.apt_url
server.pypi_index_url = env.pypi_index_url
server.veil_home = env.veil_home
server.code_dir = server.veil_home.parent
server.veil_framework_home = server.code_dir / 'veil'
server.container_name = '{}---{}'.format(server.VEIL_ENV.name, server.name)
server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name)
server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path)
server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name)
server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name)
server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name)
server.host = None
env.server_list.append(server)
env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
for host_name, host in env.hosts.items():
assert NAME_PATTERN.match(host_name) is not None, 'invalid characters in veil host name: {}'.format(host_name)
host.VEIL_ENV = env.VEIL_ENV
host.name = host_name
# host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name))
# 生产环境部署到多个host，staging只有一台host，用一台host模拟多台host时，在staging的host name加--1,--2,--3表示多台机器
host.base_name = host.name.rsplit('--', 1)[0]
host.apt_url = env.apt_url
host.pypi_index_url = env.pypi_index_url
host.ssh_user_home = as_path('/home') / host.ssh_user
host.share_dir = SHARE_DIR
host.env_dir = env.env_dir
host.etc_dir = host.env_dir / 'etc'
host.log_dir = host.env_dir / 'log'
host.var_dir = host.env_dir / 'var'
host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial'
host.buckets_dir = host.var_dir / 'buckets'
host.bucket_log_dir = host.buckets_dir / 'log'
host.data_dir = host.var_dir / 'data'
host.barman_dir = host.var_dir / 'barman'
host.veil_home = env.veil_home
host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name)
host.code_dir = host.veil_home.parent
host.veil_framework_home = host.code_dir / 'veil'
host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name)
host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path)
host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name)
host.initialized_tag_link = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.base_name)
host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name)
host.with_user_editor = False
host.server_list = []
for server_name, server in env.servers.items():
if host.name != server.host_name:
continue
server.host = host
server.lxd_endpoint = host.lxd_endpoint
server.host_base_name = host.base_name
server.ssh_user = host.ssh_user
server.ssh_port = host.ssh_port
server.ssh_user_group = host.ssh_user_group
server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no)
server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port)
server.env_dir = host.env_dir
server.etc_dir = host.etc_dir / server.name
server.log_dir = host.log_dir / server.name
server.var_dir = host.var_dir if server.mount_var_dir else None
server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None
server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None
server.data_dir = host.data_dir if server.mount_data_dir else None
server.barman_dir = host.barman_dir if server.mount_barman_dir else None
host.with_user_editor = host.with_user_editor or server.mount_editorial_dir
host.server_list.append(server)
host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
if env.hosts:
assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name)
assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name)
assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in
env.hosts.values()), 'ENV {}: found sequence no conflict among servers on one host'.format(env.name)
assert all(len([server for server in servers if server.is_guard]) <= 1 for _, servers
in itertools.groupby(sorted(env.servers.values(), key=operator.attrgetter('host_base_name')), operator.attrgetter('host_base_name'))
), 'ENV {}: found more than one guard on one host'.format(env.name)
assert len([server for server in env.servers.values() if server.is_barman]) <= 1, 'ENV {}: found more than one barman'.format(env.name)
assert len([server for server in env.servers.values() if server.is_monitor]) <= 1, 'ENV {}: found more than one monitor'.format(env.name)
assert all(not server.is_guard or server.var_dir for server in env.servers.values()), 'ENV {}: found guard without var mount'.format(env.name)
assert all(not server.is_barman or server.barman_dir for server in env.servers.values()), 'ENV {}: found barman without barman mount'.format(env.name)
assert all(server.is_guard or not server.var_dir for server in env.servers.values()), 'ENV {}: found non-guard servers with var mount'.format(env.name)
assert all(
not server.is_monitor or not server.var_dir and not server.editorial_dir and not server.buckets_dir and not server.data_dir and not server.barman_dir
for server in env.servers.values()), 'ENV {}: found monitor with var/editorial/buckets/data mount'.format(env.name)
assert all(
(server.is_guard or server.is_barman) and server.backup_mirror or not (server.is_guard or server.is_barman) and not server.backup_mirror for server
in env.servers.values()), 'ENV {}: found guard/barman without backup mirror or non-guard/barman server with backup mirror'.format(env.name)
assert all(not server.backup_mirror or env.backup_mirror == server.backup_mirror for server in env.servers.values()), \
'ENV {}: found more than one backup mirror, at most one is allowed in one env.'.format(env.name)
# break cyclic reference between host and server to get freeze_dict_object out of complain
for server in env.servers.values():
del server.host
return env
def veil_env(name, hosts, servers, sorted_server_names=None, apt_url=APT_URL, pypi_index_url=PYPI_INDEX_URL, deployment_memo=None, config=None):
server_names = servers.keys()
if sorted_server_names:
assert set(sorted_server_names) == set(server_names), \
'ENV {}: inconsistency between sorted_server_names {} and server_names {}'.format(name, sorted_server_names, server_names)
assert '@monitor' not in sorted_server_names or '@monitor' == sorted_server_names[-1], \
'ENV {}: @monitor should be the last one in {}'.format(name, sorted_server_names)
else:
sorted_server_names = sorted(server_names)
from veil.model.collection import objectify
env = objectify({
'name': name, 'hosts': hosts, 'servers': servers, 'sorted_server_names': sorted_server_names,
'apt_url': apt_url, 'pypi_index_host': urlparse(pypi_index_url).hostname, 'pypi_index_url': pypi_index_url,
'deployment_memo': deployment_memo, 'config': config or {}
})
env.VEIL_ENV = VeilEnv(env.name)
env.env_dir = OPT_DIR / env.name
env.veil_home = VEIL_HOME if env.VEIL_ENV.is_dev or env.VEIL_ENV.is_test else env.env_dir / 'code' / 'app'
env.server_list = []
for server_name, server in env.servers.items():
server.VEIL_ENV = env.VEIL_ENV
server.name = server_name
server.fullname = '{}/{}'.format(server.VEIL_ENV.name, server.name)
server.start_order = 1000 + 10 * sorted_server_names.index(server.name) if sorted_server_names else 0
server.apt_url = env.apt_url
server.pypi_index_host = env.pypi_index_host
server.pypi_index_url = env.pypi_index_url
server.veil_home = env.veil_home
server.code_dir = server.veil_home.parent
server.veil_framework_home = server.code_dir / 'veil'
server.container_name = '{}-{}'.format(server.VEIL_ENV.name, server.name)
server.container_installer_path = SHARE_DIR / 'veil-container-INSTALLER-{}'.format(server.container_name)
server.installed_container_installer_path = '{}.installed'.format(server.container_installer_path)
server.container_initialized_tag_path = SHARE_DIR / 'veil-container-{}.initialized'.format(server.container_name)
server.deployed_tag_path = SHARE_DIR / 'veil-server-{}.deployed'.format(server.container_name)
server.patched_tag_path = SHARE_DIR / 'veil-server-{}.patched'.format(server.container_name)
server.host = None
env.server_list.append(server)
env.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
for host_name, host in env.hosts.items():
host.VEIL_ENV = env.VEIL_ENV
host.name = host_name
# host base_name can be used to determine host config dir: as_path('{}/{}/hosts/{}'.format(config_dir, host.VEIL_ENV.name, host.base_name))
host.base_name = host.name.split('/', 1)[0] # e.g. ljhost-90/1 => ljhost-90
host.apt_url = env.apt_url
host.pypi_index_host = env.pypi_index_host
host.pypi_index_url = env.pypi_index_url
host.ssh_user_home = as_path('/home') / host.ssh_user
host.opt_dir = OPT_DIR
host.share_dir = SHARE_DIR
host.env_dir = env.env_dir
host.etc_dir = host.env_dir / 'etc'
host.log_dir = host.env_dir / 'log'
host.var_dir = host.env_dir / 'var'
host.editorial_dir = host.var_dir / 'editor-rootfs' / 'editorial'
host.buckets_dir = host.var_dir / 'buckets'
host.bucket_log_dir = host.buckets_dir / 'log'
host.bucket_inline_static_files_dir = host.buckets_dir / 'inline-static-files'
host.bucket_captcha_image_dir = host.buckets_dir / 'captcha-image'
host.bucket_uploaded_files_dir = host.buckets_dir / 'uploaded-files'
host.data_dir = host.var_dir / 'data'
host.veil_home = env.veil_home
host.veil_application_branch = 'env-{}'.format(host.VEIL_ENV.name)
host.code_dir = host.veil_home.parent
host.veil_framework_home = host.code_dir / 'veil'
host.iptables_rules_installer_path = SHARE_DIR / 'veil-host-iptables-rules-INSTALLER-{}'.format(host.VEIL_ENV.name)
host.installed_iptables_rules_installer_path = '{}.installed'.format(host.iptables_rules_installer_path)
host.initialized_tag_path = SHARE_DIR / 'veil-host-{}.initialized'.format(host.VEIL_ENV.name)
host.rollbackable_tag_path = SHARE_DIR / 'veil-host-{}.rollbackable'.format(host.VEIL_ENV.name)
host.with_user_editor = False
host.server_list = []
for server_name, server in env.servers.items():
if host.name != server.host_name:
continue
server.host = host
server.host_base_name = host.base_name
server.ssh_user = host.ssh_user
server.ssh_port = server.ssh_port or host.ssh_port
server.ssh_user_group = host.ssh_user_group
server.internal_ip = '{}.{}'.format(host.lan_range, server.sequence_no)
server.deploys_via = '{}@{}:{}'.format(server.ssh_user, server.internal_ip, server.ssh_port)
if server.backup_mirror:
assert server.backup_mirror.host_ip.rsplit('.', 1)[0] != host.lan_range, \
'ENV {}: local backup mirror does not make sense, please use remote mirror (on-site must, off-site optional)'.format(env.name)
server.etc_dir = host.etc_dir / server.name
server.log_dir = host.log_dir / server.name
server.editorial_dir = host.editorial_dir if server.mount_editorial_dir else None
server.buckets_dir = host.buckets_dir if server.mount_buckets_dir else None
server.data_dir = host.data_dir if server.mount_data_dir else None
host.with_user_editor = host.with_user_editor or server.mount_editorial_dir
host.server_list.append(server)
host.server_list.sort(key=lambda s: env.sorted_server_names.index(s.name))
if env.hosts:
assert all(host.server_list for host in env.hosts.values()), 'ENV {}: found host without server(s)'.format(env.name)
assert all(server.host for server in env.servers.values()), 'ENV {}: found server without host'.format(env.name)
assert all(len(host.server_list) == len(set(server.sequence_no for server in host.server_list)) for host in env.hosts.values()), \
'ENV {}: found sequence no conflict among servers on one host'.format(env.name)
assert all(
server.name not in ('@guard', '@monitor') or not server.mount_editorial_dir and not server.mount_buckets_dir and not server.mount_data_dir
for server in env.servers.values()), 'ENV {}: found @guard or @monitor with editorial/buckets/data mount'.format(env.name)
# break cyclic reference between host and server to get freeze_dict_object out of complain
for server in env.servers.values():
del server.host
return env
