def ensure(**context): if not 'CURRENT_VIRTUAL_ENV_DIR' in env: raise Exception(u'只可以在虚拟环境安装Python包') venv_dir = env.CURRENT_VIRTUAL_ENV_DIR package.ensure('supervisor') context.setdefault('run_root', venv_dir) context.setdefault('username', util.random_str(10)) context.setdefault('password', util.random_str(20, True)) context.setdefault('process_count', 2) context.setdefault('venv_dir', venv_dir) context.setdefault('virtualenv_name', venv_dir[-1:]) if 'VENV_PORT_PREFIX_MAP' in env and isinstance(env.VENV_PORT_PREFIX_MAP, dict): try: context.setdefault('port', env.VENV_PORT_PREFIX_MAP[venv_dir[-1:]]) except KeyError: raise Exception(u'你的端口配置VENV_DIR_PORT_MAP中key[%s]不存在!' % venv_dir[-1:]) if 'PROCESS_COUNT' in env: context.setdefault('process_count', env.PROCESS_COUNT) config.check('SUPERVISOR_CONF_TEMPLATE') config_template = env.SUPERVISOR_CONF_TEMPLATE destination = path.join(venv_dir, 'etc', 'supervisord.conf') template_dir, filename = path.dirname(config_template), path.basename(config_template) files.upload_template(filename, destination, context=context, use_jinja=True, template_dir=template_dir)
def get_next_version(package_name=None): """计算下一个版本号""" if not package_name: config.check('PROJECT') package_name = env.PROJECT now = datetime.datetime.now() prefix = '%s.%s.%s' % (str(now.year)[-1], now.month, now.day) latest_version = get_latest_version(package_name) #如果该项目没有建立过版本,从1开始 if not latest_version: index = 1 else: last_prefix, last_index = latest_version.rsplit('.', 1) if last_prefix != prefix: index = 1 else: index = int(last_index) + 1 version = prefix + '.' + str(index) print u'下一个版本: ' + version return version
def get_next_version(package_name=None): """计算下一个版本号""" if not package_name: config.check('PROJECT') package_name = env.PROJECT now = datetime.datetime.now() prefix = '%s.%s.%s' % (str(now.year)[-1], now.month, now.day) latest_version = get_latest_version(package_name) #如果该项目没有建立过版本,从1开始 if not latest_version: index = 1 else: last_prefix, last_index = latest_version.rsplit('.', 1) if last_prefix != prefix: index = 1 else: index = int(last_index) + 1 version = prefix + '.' + str(index) print u'下一个版本: ' + version return version
def ensure(**context): if not 'CURRENT_VIRTUAL_ENV_DIR' in env: raise Exception(u'只可以在虚拟环境安装Python包') venv_dir = env.CURRENT_VIRTUAL_ENV_DIR package.ensure('supervisor') context.setdefault('run_root', venv_dir) context.setdefault('username', util.random_str(10)) context.setdefault('password', util.random_str(20, True)) context.setdefault('process_count', 2) context.setdefault('venv_dir', venv_dir) context.setdefault('virtualenv_name', venv_dir[-1:]) if 'VENV_PORT_PREFIX_MAP' in env and isinstance(env.VENV_PORT_PREFIX_MAP, dict): try: context.setdefault('port', env.VENV_PORT_PREFIX_MAP[venv_dir[-1:]]) except KeyError: raise Exception(u'你的端口配置VENV_DIR_PORT_MAP中key[%s]不存在!' % venv_dir[-1:]) if 'PROCESS_COUNT' in env: context.setdefault('process_count', env.PROCESS_COUNT) config.check('SUPERVISOR_CONF_TEMPLATE') config_template = env.SUPERVISOR_CONF_TEMPLATE destination = path.join(venv_dir, 'etc', 'supervisord.conf') template_dir, filename = path.dirname(config_template), path.basename( config_template) files.upload_template(filename, destination, context=context, use_jinja=True, template_dir=template_dir)
def sync(*packages): """从http://pypi.python.org同步包 用法: fab pypi.sync:django==1.3,tornado """ config.check('PYPI_HOST', 'PYPI_USER', 'PYPI_ROOT') with settings(host_string=env.PYPI_HOST, user=env.PYPI_USER): cmd = ["pip", "-q", "install", "--no-deps", "-i", "https://pypi.python.org/simple", "-d", env.PYPI_ROOT, ' '.join(packages)] run(" ".join(cmd))
def sync(*packages): """从http://pypi.python.org同步包 用法: fab pypi.sync:django==1.3,tornado """ config.check('PYPI_HOST', 'PYPI_USER', 'PYPI_ROOT') with settings(host_string=env.PYPI_HOST, user=env.PYPI_USER): cmd = [ "pip", "-q", "install", "--no-deps", "-i", "https://pypi.python.org/simple", "-d", env.PYPI_ROOT, ' '.join(packages) ] run(" ".join(cmd))
def _nginx_command(command, nginx_bin=None, nginx_conf=None, use_sudo=False): if not nginx_bin: config.check('NGINX_BIN') nginx_bin = env.NGINX_BIN if not nginx_conf: config.check('NGINX_CONF') nginx_conf = env.NGINX_CONF if command == 'start': cmd = '%(nginx_bin)s -c %(nginx_conf)s' % locals() else: cmd = '%(nginx_bin)s -c %(nginx_conf)s -s %(command)s' % locals() if use_sudo: sudo(cmd) else: run(cmd)
def _nginx_command(command, nginx_bin=None, nginx_conf=None, use_sudo=False): if not nginx_bin: config.check('NGINX_BIN') nginx_bin = env.NGINX_BIN if not nginx_conf: config.check('NGINX_CONF') nginx_conf = env.NGINX_CONF if command == 'start': cmd = '%(nginx_bin)s -c %(nginx_conf)s' % locals() else: cmd = '%(nginx_bin)s -c %(nginx_conf)s -s %(command)s' % locals() if use_sudo: sudo(cmd) else: run(cmd)
def switch(src_pattern, dst_pattern, root=None, nginx_bin=None, nginx_conf=None): """ 修改配置文件并重启:源文本,目标文本,[root](使用root) 主要用于AB环境的切换,将配置文件中的src_pattern修改为dst_pattern,并重启。 参数: src_pattern: 源模式,如upstreamA src_pattern: 目标模式,如upstreamB nginx_bin: nginx可执行文件路径,如果为提供则从env获取。 nginx_conf: nginx配置文件路径,如果为提供则从env获取。 """ if not nginx_conf: config.check('NGINX_CONF') nginx_conf = env.NGINX_CONF use_sudo = (root=='root') files.sed(nginx_conf, src_pattern, dst_pattern, use_sudo=use_sudo) reload(nginx_bin, nginx_conf, use_sudo=use_sudo)
def get_latest_version(package_name=None): if not package_name: config.check('PROJECT') package_name = env.PROJECT # 这里直接使用了package finder,而非search command, # 是因为pypiserver不支持pip search from pip.index import PackageFinder from pip.req import InstallRequirement finder = PackageFinder(find_links=[], index_urls=[env.PYPI_INDEX]) req = InstallRequirement(req=package_name, comes_from=None) try: url = finder.find_requirement(req, upgrade=True) except DistributionNotFound: print u'尚无任何版本!' return None filename = url.splitext()[0] version = re.search(r'(\d+\.?)+', filename) version = version.group() if version else None print u'当前版本: %s' % version return version
def get_latest_version(package_name=None): if not package_name: config.check('PROJECT') package_name = env.PROJECT # 这里直接使用了package finder,而非search command, # 是因为pypiserver不支持pip search from pip.index import PackageFinder from pip.req import InstallRequirement finder = PackageFinder(find_links=[], index_urls=[env.PYPI_INDEX]) req = InstallRequirement(req=package_name, comes_from=None) try: url = finder.find_requirement(req, upgrade=True) except DistributionNotFound: print u'尚无任何版本!' return None filename = url.splitext()[0] version = re.search(r'(\d+\.?)+', filename) version = version.group() if version else None print u'当前版本: %s' % version return version
def switch(src_pattern, dst_pattern, root=None, nginx_bin=None, nginx_conf=None): """ 修改配置文件并重启:源文本,目标文本,[root](使用root) 主要用于AB环境的切换,将配置文件中的src_pattern修改为dst_pattern,并重启。 参数: src_pattern: 源模式,如upstreamA src_pattern: 目标模式,如upstreamB nginx_bin: nginx可执行文件路径,如果为提供则从env获取。 nginx_conf: nginx配置文件路径,如果为提供则从env获取。 """ if not nginx_conf: config.check('NGINX_CONF') nginx_conf = env.NGINX_CONF use_sudo = (root == 'root') files.sed(nginx_conf, src_pattern, dst_pattern, use_sudo=use_sudo) reload(nginx_bin, nginx_conf, use_sudo=use_sudo)
def main( cli_ctx: click.Context, config_path: Path, debug: bool, ) -> int: # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent') # Override the read config with environment variables (for legacy). config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'host'), 'BACKEND_AGENT_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'port'), 'BACKEND_AGENT_PORT') config.override_with_env(raw_cfg, ('agent', 'pid-file'), 'BACKEND_PID_FILE') config.override_with_env(raw_cfg, ('container', 'port-range'), 'BACKEND_CONTAINER_PORT_RANGE') config.override_with_env(raw_cfg, ('container', 'kernel-host'), 'BACKEND_KERNEL_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('container', 'sandbox-type'), 'BACKEND_SANDBOX_TYPE') config.override_with_env(raw_cfg, ('container', 'scratch-root'), 'BACKEND_SCRATCH_ROOT') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) config.override_key(raw_cfg, ('logging', 'level'), 'DEBUG') config.override_key(raw_cfg, ('logging', 'pkg-ns', 'ai.backend'), 'DEBUG') # Validate and fill configurations # (allow_extra will make configs to be forward-copmatible) try: cfg = config.check(raw_cfg, agent_local_config_iv) if cfg['agent']['backend'] == AgentBackend.KUBERNETES: cfg = config.check(raw_cfg, k8s_extra_config_iv) if cfg['registry']['type'] == 'local': registry_target_config_iv = registry_local_config_iv elif cfg['registry']['type'] == 'ecr': registry_target_config_iv = registry_ecr_config_iv else: print('Validation of agent configuration has failed: registry type {} not supported' .format(cfg['registry']['type']), file=sys.stderr) raise click.Abort() registry_cfg = config.check(cfg['registry'], registry_target_config_iv) cfg['registry'] = registry_cfg if cfg['agent']['backend'] == AgentBackend.DOCKER: config.check(raw_cfg, docker_extra_config_iv) if 'debug' in cfg and cfg['debug']['enabled']: print('== Agent configuration ==') pprint(cfg) cfg['_src'] = cfg_src_path except config.ConfigurationError as e: print('ConfigurationError: Validation of agent configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() rpc_host = cfg['agent']['rpc-listen-addr'].host if (isinstance(rpc_host, BaseIPAddress) and (rpc_host.is_unspecified or rpc_host.is_link_local)): print('ConfigurationError: ' 'Cannot use link-local or unspecified IP address as the RPC listening host.', file=sys.stderr) raise click.Abort() if os.getuid() != 0 and cfg['container']['stats-type'] == 'cgroup': print('Cannot use cgroup statistics collection mode unless the agent runs as root.', file=sys.stderr) raise click.Abort() if cli_ctx.invoked_subcommand is None: if cfg['debug']['coredump']['enabled']: if not sys.platform.startswith('linux'): print('ConfigurationError: ' 'Storing container coredumps is only supported in Linux.', file=sys.stderr) raise click.Abort() core_pattern = Path('/proc/sys/kernel/core_pattern').read_text().strip() if core_pattern.startswith('|') or not core_pattern.startswith('/'): print('ConfigurationError: ' '/proc/sys/kernel/core_pattern must be an absolute path ' 'to enable container coredumps.', file=sys.stderr) raise click.Abort() cfg['debug']['coredump']['core_path'] = Path(core_pattern).parent cfg['agent']['pid-file'].write_text(str(os.getpid())) log_sockpath = Path(f'/tmp/backend.ai/ipc/agent-logger-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' cfg['logging']['endpoint'] = log_endpoint try: logger = Logger(cfg['logging'], is_master=True, log_endpoint=log_endpoint) with logger: ns = cfg['etcd']['namespace'] setproctitle(f"backend.ai: agent {ns}") log.info('Backend.AI Agent {0}', VERSION) log.info('runtime: {0}', utils.env_info()) log_config = logging.getLogger('ai.backend.agent.config') if debug: log_config.debug('debug mode enabled.') if cfg['agent']['event-loop'] == 'uvloop': import uvloop uvloop.install() log.info('Using uvloop as the event loop backend') aiotools.start_server( server_main_logwrapper, num_workers=1, args=(cfg, log_endpoint), ) log.info('exit.') finally: if cfg['agent']['pid-file'].is_file(): # check is_file() to prevent deleting /dev/null! cfg['agent']['pid-file'].unlink() else: # Click is going to invoke a subcommand. pass return 0
def build(name=None, version=None, commit=None, branch=None): """ 打包 参数: name: 描述, 如:seo。最后生成project_name-x.x.x.x-seo.tar.gz commit: 指定commit版本 branch: 分支名称 version: 自定义版本号,如果为None则根据日期生成 commit和branch必须提供一个, 或者读取配置文件 """ if commit: check_out = commit elif branch: check_out = branch else: check_out = env.DEFAULT_BRANCH if not version: config.check('PROJECT') version = get_next_version(env.PROJECT) if name: version = '%s-%s' % (version, name) project_path = os.path.join(env.BUILD_PATH, env.PROJECT) if not files.exists(project_path): with(cd(env.BUILD_PATH)): git.clone('/'.join([env.PROJECT_OWNER, env.PROJECT])) with(cd(project_path)): git.checkout(check_out) # 在setup打包之前做进一步数据准备工作的hook if hasattr(env, 'PRE_BUILD_HOOK'): env.PRE_BUILD_HOOK() params = { 'release_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'git_version': git.get_version(), 'version': version, } fs.inplace_render(os.path.join(project_path, 'setup.py'), params) if hasattr(env, 'SETTINGS_BASE_FILE'): settings_file_path = os.path.join(project_path, *env.SETTINGS_BASE_FILE.split('/')) if files.exists(settings_file_path): fs.inplace_render(settings_file_path, params) else: settings_file_path = os.path.join(project_path, env.PROJECT, 'settings.py') if files.exists(settings_file_path): fs.inplace_render(settings_file_path, params) settings_dir_path = os.path.join(project_path, env.PROJECT, 'settings', '__init__.py') if files.exists(settings_dir_path): fs.inplace_render(settings_dir_path, params) run("python setup.py sdist upload -r internal")
def build(name=None, version=None, commit=None, branch=None): """ 打包 参数: name: 描述, 如:seo。最后生成project_name-x.x.x.x-seo.tar.gz commit: 指定commit版本 branch: 分支名称 version: 自定义版本号,如果为None则根据日期生成 commit和branch必须提供一个, 或者读取配置文件 """ if commit: check_out = commit elif branch: check_out = branch else: check_out = env.DEFAULT_BRANCH if not version: config.check('PROJECT') version = get_next_version(env.PROJECT) if name: version = '%s-%s' % (version, name) project_path = os.path.join(env.BUILD_PATH, env.PROJECT) if not files.exists(project_path): with (cd(env.BUILD_PATH)): git.clone('/'.join([env.PROJECT_OWNER, env.PROJECT])) with (cd(project_path)): git.checkout(check_out) # 在setup打包之前做进一步数据准备工作的hook if hasattr(env, 'PRE_BUILD_HOOK'): env.PRE_BUILD_HOOK() params = { 'release_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'git_version': git.get_version(), 'version': version, } fs.inplace_render(os.path.join(project_path, 'setup.py'), params) if hasattr(env, 'SETTINGS_BASE_FILE'): settings_file_path = os.path.join( project_path, *env.SETTINGS_BASE_FILE.split('/')) if files.exists(settings_file_path): fs.inplace_render(settings_file_path, params) else: settings_file_path = os.path.join(project_path, env.PROJECT, 'settings.py') if files.exists(settings_file_path): fs.inplace_render(settings_file_path, params) settings_dir_path = os.path.join(project_path, env.PROJECT, 'settings', '__init__.py') if files.exists(settings_dir_path): fs.inplace_render(settings_dir_path, params) run("python setup.py sdist upload -r internal")