def chart(tar_dir: PATH_TYPE = "./"): """ cli alias: ``arch chart`` Parameters ---------- tar_dir: target directory """ src_dir = path_append(META, "chart") tar_dir = path_append(tar_dir, "chart/") logger.info("chart: copy %s -> %s" % (src_dir, tar_dir)) copytree(src_dir, tar_dir)
def nni(tar_dir="./"): """ cli alias: ``arch nni`` and ``install nni`` Parameters ---------- tar_dir Returns ------- """ src_dir = path_append(META, "nni") for file in ["_config.yml", "_search_space.json"]: copyfile(path_append(src_dir, file), path_append(tar_dir, file))
def gitignore(atype: str = "", tar_dir: PATH_TYPE = "./"): """ cli alias: ``arch gitignore`` Parameters ---------- atype: the gitignore type, currently support `docs` and `python` tar_dir: target directory """ src = path_append(META, "gitignore", "%s.gitignore" % atype) tar = path_append(tar_dir, ".gitignore") logger.info("gitignore: copy %s -> %s" % (src, tar)) copyfile(src, tar)
def readthedocs(tar_dir="./"): """ cli alias: ``arch readthedocs`` Parameters ---------- tar_dir Returns ------- """ src = path_append(META, "docs/.readthedocs.yml") tar = path_append(tar_dir, ".readthedocs.yml") logger.info("readthedocs: copy %s -> %s" % (src, tar)) copyfile(src, tar)
def pytest(tar_dir: PATH_TYPE = "./"): """ cli alias: ``arch pytest`` Parameters ---------- tar_dir Returns ------- """ src = path_append(META, "pytest.ini") tar = path_append(tar_dir, "pytest.ini") logger.info("pytest: copy %s -> %s" % (src, tar)) copyfile(src, tar)
def travis(tar_dir: PATH_TYPE = "./"): """ cli alias: ``arch travis`` Parameters ---------- tar_dir Returns ------- """ src = path_append(META, ".travis.yml") tar = path_append(tar_dir, ".travis.yml") logger.info("travis: copy %s -> %s" % (src, tar)) copyfile(src, tar)
def pysetup(tar_dir="./", **variables): """ cli alias: ``arch pysetup`` Parameters ---------- tar_dir variables Returns ------- """ src = path_append(META, "setup.py.template") tar = path_append(tar_dir, "setup.py") logger.info("pysetup: template %s -> %s" % (src, tar)) template_copy(src, tar, **variables)
def coverage(tar_dir: PATH_TYPE = "./", **variables): """ cli alias: ``arch coverage`` Parameters ---------- tar_dir: variables: These variables should be provided: * project """ src = path_append(META, "setup.cfg.template") tar = path_append(tar_dir, "setup.cfg") logger.info("coverage: template %s -> %s" % (src, tar)) template_copy(src, tar, quotation="", **variables)
def makefile(tar_dir="./", **variables): """ cli alias: ``arch makefile`` Parameters ---------- tar_dir variables Returns ------- """ src = path_append(META, "Makefile.template") tar = path_append(tar_dir, "Makefile") logger.info("makefile: template %s -> %s" % (src, tar)) template_copy(src, tar, default_value=None, quotation='', **variables)
def dockerfile(atype, tar_dir="./", **variables): """ cli alias: ``arch dockerfile`` Parameters ---------- atype tar_dir variables Returns ------- """ src = path_append(META, "Dockerfile/%s.docker" % atype) tar = path_append(tar_dir, "Dockerfile") logger.info("Dockerfile: %s -> %s" % (src, tar)) template_copy(src, tar, quotation="", **variables)
def helm_service(host="${KUBE_NAMESPACE}", image_repo="${CI_REGISTRY_IMAGE}", image_port=None, private=True, name="$KUBE_NAMESPACE", image_tag="latest", path_to_api=""): """ Parameters ---------- host image_repo image_port private name image_tag path_to_api Returns ------- """ src = path_append(META, "gitlab-ci/helm_install.gitlab-ci.yml.template") variables = { "NAME": name, "IMAGE_TAG": image_tag, "IMAGE_REPO": image_repo, "HOST": host, "PATH_TO_API": path_to_api, } with open(src, encoding="utf-8") as f: helm_install_commands = "".join(f.readlines()).strip() + "\n" helm_install_commands = default_variable_replace(helm_install_commands, key_lower=False, quotation="", **variables) if image_port: helm_install_commands += "--set \"image.port=%s\"" % image_port + "\n" if private: helm_install_commands += "--set \"dockercfg=$(cat /root/.docker/config.json | base64 | tr -d '\\n')\"" + "\n" if not path_to_api: helm_install_commands += "--set \"ingress.annotations=null\"" + "\n" return FoldedString(helm_install_commands)
def sphinx_conf(tar_dir="./", **variables): """ cli alias: ``arch sphinx_conf`` Parameters ---------- tar_dir variables Returns ------- """ if variables["docs_style"] == "mxnet": src = path_append(META, "docs/mxnet/conf.py.template") tar = path_append(tar_dir, "conf.py") logger.info("sphinx_conf: template %s -> %s" % (src, tar)) template_copy(src, tar, **variables) src = path_append(META, "docs/mxnet/.math.json") tar = path_append(tar_dir, ".math.json") logger.info("sphinx_conf: copy %s -> %s" % (src, tar)) copyfile(src, tar) src = path_append(META, "docs/mxnet/requirements.txt") tar = path_append(tar_dir, "requirements.txt") logger.info("sphinx_conf: copy %s -> %s" % (src, tar)) copyfile(src, tar) logger.warning( "\n%s\nmodify setup.py according to the components in %s\n%s\n" % ('*' * 60, tar, '*' * 60)) else: src = path_append(META, "docs/sphinx/requirements.txt") tar = path_append(tar_dir, "requirements.txt") logger.info("sphinx_conf: copy %s -> %s" % (src, tar)) copyfile(src, tar) logger.warning( "\n%s\nmanually run 'sphinx-quickstart' in %s to create necessary components\n%s" % ('*' * 60, tar_dir, '*' * 60))
def docs_proj(tar_dir, docs_params, docker_params=None, __gitignore=True, **kwargs): assert docs_params variables = {} variables.update(docs_params) docs_root = path_append(tar_dir, docs_params["docs_root"]) if not os.path.exists(docs_root): os.makedirs(docs_root) sphinx_conf(tar_dir=docs_root, **docs_params) if docs_params["readthedocs"]: readthedocs(tar_dir=tar_dir) if __gitignore: gitignore("docs", tar_dir) if docker_params: # pragma: no cover dockerfile(docs_params["docker_type"], **docker_params, tar_dir=docs_root) elif "docker_params" in docs_params: dockerfile(docs_params["docker_params"]["docker_type"], **docs_params["docker_params"], tar_dir=docs_root)
def test_yaml(tmp_path): tmp_file = path_append(tmp_path, "test.yaml") obj = OrderedDict({ "b": 123, }) obj["a"] = 456 c = "" c += "helm install" + "\n" c += "--set \"abc\"" + "\n" obj["c"] = [FoldedString(c)] with as_out_io(tmp_file) as wf: print(dump_folded_yaml(obj), file=wf) for i, (key, value) in enumerate(ordered_yaml_load(tmp_file).items()): if i == 0: assert key == "b" and value == 123 elif i == 1: assert key == "a" and value == 456 elif i == 2: assert key == "c" and value == ['helm install --set "abc"\n']
def cli(skip_top=True, project=None, override=None, tar_dir="./", **kwargs): """ The main function for arch """ config.OVERRIDE = override main_params = main_cli(skip_top, project, **kwargs) kwargs.update(main_params) indicator = dict(travis=binary_legal_input("Install travis?"), ) docs = binary_legal_input("Install docs?") if main_params["project_type"] == "docs" or docs: default_style = "mxnet" if main_params[ "project_type"] == "docs" else "sphinx" docs_params = docs_cli(default_style=default_style, **kwargs) else: docs_params = {} service_params = {} docker_params = {} if main_params["project_type"] != "docs": service = binary_legal_input("To deploy as a service") if service: docker_params.update( dockerfile_cli(project_type=main_params["project_type"])) if "port" not in docker_params: port = default_legal_input("Image Port", __default_value=None) else: port = docker_params["port"] service_params["port"] = None if port == 'null' else port service_params.update( dict(private=binary_legal_input("Is private project?"), )) stages_candidates = { "test": { "stage_image_name": docker_params["image_name"] } } if main_params["project_type"] == "python": stages_candidates["build"] = {"need": "n"} if binary_legal_input("Install .gitlab-ci.yml?"): service_params.update( dict(gitlab_ci_params=gitlab_ci_cli( port=port, docs=docs, stages_candidates=stages_candidates), )) elif binary_legal_input("Install Dockerfile?"): docker_params.update( dockerfile_cli(project_type=main_params["project_type"])) if skip_top: tar_dir = path_append(tar_dir, "./") else: tar_dir = path_append(tar_dir, main_params["project"]) os.makedirs(tar_dir) __project_type = main_params["project_type"] project_types[__project_type](tar_dir=tar_dir, main_params=main_params, docs_params=docs_params, docker_params=docker_params, service_params=service_params, **indicator)
# coding: utf-8 # 2020/3/10 @ tongshiwei import functools import time import random import json from longling.lib.stream import as_io from longling.lib.path import path_append, abs_current_dir from longling.lib.candylib import as_list from urllib.request import ProxyHandler, build_opener from longling.spider.conf import logger META_DATA = path_append(abs_current_dir(__file__), '../meta_data/') HTTP_HEADER = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36', 'Connection': 'keep-alive', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'cors', } with as_io(path_append(META_DATA, 'user_agents.txt')) as f: USER_AGENTS = [line.strip() for line in f.readlines()] with as_io(path_append(META_DATA, 'IPpool.json')) as f: PROXIES = [ "%s:%s" % (ip_port["ip"], ip_port["port"]) for ip_port in json.load(f) ]
import os from collections import OrderedDict from shutil import copyfile as _copyfile, rmtree, copytree as _copytree from longling import wf_open, PATH_TYPE from longling.Architecture.utils import binary_legal_input from longling.lib.path import abs_current_dir, path_append from longling.lib.regex import default_variable_replace as dvr from longling.lib.utilog import config_logging, LogLevel from longling.lib.yaml_helper import FoldedString, ordered_yaml_load, dump_folded_yaml from . import config logger = config_logging(logger="arch", console_log_level=LogLevel.INFO) META = path_append(abs_current_dir(__file__), "meta_docs") default_variable_replace = functools.partial(dvr, quotation="\'") def copytree(src, dst, **kwargs): """ Recursively copy a directory tree. Change OVERRIDE mode to specify the operation when dst existed. Examples -------- .. code-block :: python # to change the OVERRIDE operation from longling.Architecture import config
def gitlab_ci(private, stages: dict, atype: str = "", tar_dir: PATH_TYPE = "./", version_in_path=True): """ cli alias: ``arch gitlab_ci`` Parameters ---------- private stages atype tar_dir version_in_path Returns ------- """ base_src = path_append(META, "gitlab-ci", ".gitlab-ci.yml") src = path_append(META, "gitlab-ci", "%s.gitlab-ci.yml" % atype) tar = path_append(tar_dir, ".gitlab-ci.yml") config_template = OrderedDict() with open(base_src) as f: config_template.update(ordered_yaml_load(f)) with open(src) as f: config_template.update(ordered_yaml_load(f)) logger.info("generate %s" % tar) with wf_open(tar) as wf: for _c in ["variables", "cache"]: if _c in config_template: print(dump_folded_yaml({_c: config_template[_c]}), file=wf) print(dump_folded_yaml({ "stages": [stage for stage in stages.keys() if stage in config_template] }), file=wf) for stage, params in stages.items(): if stage == "docs": params["registry_suffix"] = "/docs" elif stage in {"test", "build"}: params["deployment"] = False if stage not in config_template: logger.warning("%s is not listed in %s, skipped" % (stage, src)) continue commands = {stage: config_template[stage]} _gitlab_ci(commands, stage, private=private, version_in_path=version_in_path, **params) print(dump_folded_yaml(commands), file=wf) if private: print("*" * 30) print("私有项目注意") print( "在项目的settings->repository->Deploy Tokens 添加一个name为gitlab-deploy-token、" "其他两项留空的具有read_registry权限的token") print("*" * 30)