def chart(tar_dir: PATH_TYPE = "./"):
"""
cli alias: ``arch chart``
Parameters
----------
tar_dir: target directory
"""
src_dir = path_append(META, "chart")
tar_dir = path_append(tar_dir, "chart/")
logger.info("chart: copy %s -> %s" % (src_dir, tar_dir))
copytree(src_dir, tar_dir)
def nni(tar_dir="./"):
"""
cli alias: ``arch nni`` and ``install nni``
Parameters
----------
tar_dir
Returns
-------
"""
src_dir = path_append(META, "nni")
for file in ["_config.yml", "_search_space.json"]:
copyfile(path_append(src_dir, file), path_append(tar_dir, file))
def gitignore(atype: str = "", tar_dir: PATH_TYPE = "./"):
"""
cli alias: ``arch gitignore``
Parameters
----------
atype: the gitignore type, currently support `docs` and `python`
tar_dir: target directory
"""
src = path_append(META, "gitignore", "%s.gitignore" % atype)
tar = path_append(tar_dir, ".gitignore")
logger.info("gitignore: copy %s -> %s" % (src, tar))
copyfile(src, tar)
def readthedocs(tar_dir="./"):
"""
cli alias: ``arch readthedocs``
Parameters
----------
tar_dir
Returns
-------
"""
src = path_append(META, "docs/.readthedocs.yml")
tar = path_append(tar_dir, ".readthedocs.yml")
logger.info("readthedocs: copy %s -> %s" % (src, tar))
copyfile(src, tar)
def pytest(tar_dir: PATH_TYPE = "./"):
"""
cli alias: ``arch pytest``
Parameters
----------
tar_dir
Returns
-------
"""
src = path_append(META, "pytest.ini")
tar = path_append(tar_dir, "pytest.ini")
logger.info("pytest: copy %s -> %s" % (src, tar))
copyfile(src, tar)
def travis(tar_dir: PATH_TYPE = "./"):
"""
cli alias: ``arch travis``
Parameters
----------
tar_dir
Returns
-------
"""
src = path_append(META, ".travis.yml")
tar = path_append(tar_dir, ".travis.yml")
logger.info("travis: copy %s -> %s" % (src, tar))
copyfile(src, tar)
def pysetup(tar_dir="./", **variables):
"""
cli alias: ``arch pysetup``
Parameters
----------
tar_dir
variables
Returns
-------
"""
src = path_append(META, "setup.py.template")
tar = path_append(tar_dir, "setup.py")
logger.info("pysetup: template %s -> %s" % (src, tar))
template_copy(src, tar, **variables)
def coverage(tar_dir: PATH_TYPE = "./", **variables):
"""
cli alias: ``arch coverage``
Parameters
----------
tar_dir:
variables:
These variables should be provided:
* project
"""
src = path_append(META, "setup.cfg.template")
tar = path_append(tar_dir, "setup.cfg")
logger.info("coverage: template %s -> %s" % (src, tar))
template_copy(src, tar, quotation="", **variables)
def makefile(tar_dir="./", **variables):
"""
cli alias: ``arch makefile``
Parameters
----------
tar_dir
variables
Returns
-------
"""
src = path_append(META, "Makefile.template")
tar = path_append(tar_dir, "Makefile")
logger.info("makefile: template %s -> %s" % (src, tar))
template_copy(src, tar, default_value=None, quotation='', **variables)
def dockerfile(atype, tar_dir="./", **variables):
"""
cli alias: ``arch dockerfile``
Parameters
----------
atype
tar_dir
variables
Returns
-------
"""
src = path_append(META, "Dockerfile/%s.docker" % atype)
tar = path_append(tar_dir, "Dockerfile")
logger.info("Dockerfile: %s -> %s" % (src, tar))
template_copy(src, tar, quotation="", **variables)
def helm_service(host="${KUBE_NAMESPACE}",
image_repo="${CI_REGISTRY_IMAGE}",
image_port=None,
private=True,
name="$KUBE_NAMESPACE",
image_tag="latest",
path_to_api=""):
"""
Parameters
----------
host
image_repo
image_port
private
name
image_tag
path_to_api
Returns
-------
"""
src = path_append(META, "gitlab-ci/helm_install.gitlab-ci.yml.template")
variables = {
"NAME": name,
"IMAGE_TAG": image_tag,
"IMAGE_REPO": image_repo,
"HOST": host,
"PATH_TO_API": path_to_api,
}
with open(src, encoding="utf-8") as f:
helm_install_commands = "".join(f.readlines()).strip() + "\n"
helm_install_commands = default_variable_replace(helm_install_commands,
key_lower=False,
quotation="",
**variables)
if image_port:
helm_install_commands += "--set \"image.port=%s\"" % image_port + "\n"
if private:
helm_install_commands += "--set \"dockercfg=$(cat /root/.docker/config.json | base64 | tr -d '\\n')\"" + "\n"
if not path_to_api:
helm_install_commands += "--set \"ingress.annotations=null\"" + "\n"
return FoldedString(helm_install_commands)
def sphinx_conf(tar_dir="./", **variables):
"""
cli alias: ``arch sphinx_conf``
Parameters
----------
tar_dir
variables
Returns
-------
"""
if variables["docs_style"] == "mxnet":
src = path_append(META, "docs/mxnet/conf.py.template")
tar = path_append(tar_dir, "conf.py")
logger.info("sphinx_conf: template %s -> %s" % (src, tar))
template_copy(src, tar, **variables)
src = path_append(META, "docs/mxnet/.math.json")
tar = path_append(tar_dir, ".math.json")
logger.info("sphinx_conf: copy %s -> %s" % (src, tar))
copyfile(src, tar)
src = path_append(META, "docs/mxnet/requirements.txt")
tar = path_append(tar_dir, "requirements.txt")
logger.info("sphinx_conf: copy %s -> %s" % (src, tar))
copyfile(src, tar)
logger.warning(
"\n%s\nmodify setup.py according to the components in %s\n%s\n" %
('*' * 60, tar, '*' * 60))
else:
src = path_append(META, "docs/sphinx/requirements.txt")
tar = path_append(tar_dir, "requirements.txt")
logger.info("sphinx_conf: copy %s -> %s" % (src, tar))
copyfile(src, tar)
logger.warning(
"\n%s\nmanually run 'sphinx-quickstart' in %s to create necessary components\n%s"
% ('*' * 60, tar_dir, '*' * 60))
def docs_proj(tar_dir, docs_params, docker_params=None, __gitignore=True, **kwargs):
assert docs_params
variables = {}
variables.update(docs_params)
docs_root = path_append(tar_dir, docs_params["docs_root"])
if not os.path.exists(docs_root):
os.makedirs(docs_root)
sphinx_conf(tar_dir=docs_root, **docs_params)
if docs_params["readthedocs"]:
readthedocs(tar_dir=tar_dir)
if __gitignore:
gitignore("docs", tar_dir)
if docker_params: # pragma: no cover
dockerfile(docs_params["docker_type"], **docker_params, tar_dir=docs_root)
elif "docker_params" in docs_params:
dockerfile(docs_params["docker_params"]["docker_type"], **docs_params["docker_params"], tar_dir=docs_root)
def test_yaml(tmp_path):
tmp_file = path_append(tmp_path, "test.yaml")
obj = OrderedDict({
"b": 123,
})
obj["a"] = 456
c = ""
c += "helm install" + "\n"
c += "--set \"abc\"" + "\n"
obj["c"] = [FoldedString(c)]
with as_out_io(tmp_file) as wf:
print(dump_folded_yaml(obj), file=wf)
for i, (key, value) in enumerate(ordered_yaml_load(tmp_file).items()):
if i == 0:
assert key == "b" and value == 123
elif i == 1:
assert key == "a" and value == 456
elif i == 2:
assert key == "c" and value == ['helm install --set "abc"\n']
def cli(skip_top=True, project=None, override=None, tar_dir="./", **kwargs):
"""
The main function for arch
"""
config.OVERRIDE = override
main_params = main_cli(skip_top, project, **kwargs)
kwargs.update(main_params)
indicator = dict(travis=binary_legal_input("Install travis?"), )
docs = binary_legal_input("Install docs?")
if main_params["project_type"] == "docs" or docs:
default_style = "mxnet" if main_params[
"project_type"] == "docs" else "sphinx"
docs_params = docs_cli(default_style=default_style, **kwargs)
else:
docs_params = {}
service_params = {}
docker_params = {}
if main_params["project_type"] != "docs":
service = binary_legal_input("To deploy as a service")
if service:
docker_params.update(
dockerfile_cli(project_type=main_params["project_type"]))
if "port" not in docker_params:
port = default_legal_input("Image Port", __default_value=None)
else:
port = docker_params["port"]
service_params["port"] = None if port == 'null' else port
service_params.update(
dict(private=binary_legal_input("Is private project?"), ))
stages_candidates = {
"test": {
"stage_image_name": docker_params["image_name"]
}
}
if main_params["project_type"] == "python":
stages_candidates["build"] = {"need": "n"}
if binary_legal_input("Install .gitlab-ci.yml?"):
service_params.update(
dict(gitlab_ci_params=gitlab_ci_cli(
port=port,
docs=docs,
stages_candidates=stages_candidates), ))
elif binary_legal_input("Install Dockerfile?"):
docker_params.update(
dockerfile_cli(project_type=main_params["project_type"]))
if skip_top:
tar_dir = path_append(tar_dir, "./")
else:
tar_dir = path_append(tar_dir, main_params["project"])
os.makedirs(tar_dir)
__project_type = main_params["project_type"]
project_types[__project_type](tar_dir=tar_dir,
main_params=main_params,
docs_params=docs_params,
docker_params=docker_params,
service_params=service_params,
**indicator)
# coding: utf-8
# 2020/3/10 @ tongshiwei
import functools
import time
import random
import json
from longling.lib.stream import as_io
from longling.lib.path import path_append, abs_current_dir
from longling.lib.candylib import as_list
from urllib.request import ProxyHandler, build_opener
from longling.spider.conf import logger
META_DATA = path_append(abs_current_dir(__file__), '../meta_data/')
HTTP_HEADER = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
'Connection': 'keep-alive',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
}
with as_io(path_append(META_DATA, 'user_agents.txt')) as f:
USER_AGENTS = [line.strip() for line in f.readlines()]
with as_io(path_append(META_DATA, 'IPpool.json')) as f:
PROXIES = [
"%s:%s" % (ip_port["ip"], ip_port["port"]) for ip_port in json.load(f)
]
import os
from collections import OrderedDict
from shutil import copyfile as _copyfile, rmtree, copytree as _copytree
from longling import wf_open, PATH_TYPE
from longling.Architecture.utils import binary_legal_input
from longling.lib.path import abs_current_dir, path_append
from longling.lib.regex import default_variable_replace as dvr
from longling.lib.utilog import config_logging, LogLevel
from longling.lib.yaml_helper import FoldedString, ordered_yaml_load, dump_folded_yaml
from . import config
logger = config_logging(logger="arch", console_log_level=LogLevel.INFO)
META = path_append(abs_current_dir(__file__), "meta_docs")
default_variable_replace = functools.partial(dvr, quotation="\'")
def copytree(src, dst, **kwargs):
"""
Recursively copy a directory tree.
Change OVERRIDE mode to specify the operation when dst existed.
Examples
--------
.. code-block :: python
# to change the OVERRIDE operation
from longling.Architecture import config
def gitlab_ci(private,
stages: dict,
atype: str = "",
tar_dir: PATH_TYPE = "./",
version_in_path=True):
"""
cli alias: ``arch gitlab_ci``
Parameters
----------
private
stages
atype
tar_dir
version_in_path
Returns
-------
"""
base_src = path_append(META, "gitlab-ci", ".gitlab-ci.yml")
src = path_append(META, "gitlab-ci", "%s.gitlab-ci.yml" % atype)
tar = path_append(tar_dir, ".gitlab-ci.yml")
config_template = OrderedDict()
with open(base_src) as f:
config_template.update(ordered_yaml_load(f))
with open(src) as f:
config_template.update(ordered_yaml_load(f))
logger.info("generate %s" % tar)
with wf_open(tar) as wf:
for _c in ["variables", "cache"]:
if _c in config_template:
print(dump_folded_yaml({_c: config_template[_c]}), file=wf)
print(dump_folded_yaml({
"stages":
[stage for stage in stages.keys() if stage in config_template]
}),
file=wf)
for stage, params in stages.items():
if stage == "docs":
params["registry_suffix"] = "/docs"
elif stage in {"test", "build"}:
params["deployment"] = False
if stage not in config_template:
logger.warning("%s is not listed in %s, skipped" %
(stage, src))
continue
commands = {stage: config_template[stage]}
_gitlab_ci(commands,
stage,
private=private,
version_in_path=version_in_path,
**params)
print(dump_folded_yaml(commands), file=wf)
if private:
print("*" * 30)
print("私有项目注意")
print(
"在项目的settings->repository->Deploy Tokens 添加一个name为gitlab-deploy-token、"
"其他两项留空的具有read_registry权限的token")
print("*" * 30)
