def test_save_configuration(tmp_path):
configuration_file = tmp_path.joinpath("zebr0.conf")
client = zebr0.Client("http://127.0.0.1:8000", levels=["lorem", "ipsum"], cache=1, configuration_file=configuration_file)
client.save_configuration()
assert configuration_file.read_text(zebr0.ENCODING) == '{"url": "http://127.0.0.1:8000", "levels": ["lorem", "ipsum"], "cache": 1}'
def test_read_ok(tmp_path, server):
file = tmp_path.joinpath("file")
file.write_text("content")
server.data = {"template": "{{ '" + str(file) + "' | read }}"}
client = zebr0.Client("http://127.0.0.1:8000", configuration_file=Path(""))
assert client.get("template") == "content"
def test_recursive_render(server):
server.data = {
"answer": "42",
"template": "the answer is {{ 'answer' | get }}"
}
client = zebr0.Client("http://127.0.0.1:8000", configuration_file=Path(""))
assert client.get("template") == "the answer is 42"
def test_configuration_file(server, tmp_path):
configuration_file = tmp_path.joinpath("zebr0.conf")
configuration_file.write_text('{"url": "http://127.0.0.1:8000", "levels": ["lorem", "ipsum"], "cache": 1}', zebr0.ENCODING)
server.data = {"lorem/ipsum/dolor": "sit amet"}
client = zebr0.Client(configuration_file=configuration_file)
assert client.get("dolor") == "sit amet"
def test_get_latest_image_fixed(server):
server.data = {
"aws-region": region,
"aws-ami-criteria": GET_LATEST_IMAGE_FIXED_CRITERIA
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
assert client.get_latest_image().get("ImageId") == "ami-06ec8443c2a35b0ba"
def test_get_latest_image_wrong(server):
server.data = {
"aws-region": region,
"aws-ami-criteria": GET_LATEST_IMAGE_WRONG_CRITERIA
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
assert client.get_latest_image() is None
def test_recursive_nested_render(server):
server.data = {
"what": "memes",
"star_wars/what": "droids",
"star_wars/punctuation_mark": ".",
"star_wars/slang/punctuation_mark": ", duh!",
"template": "these aren't the {{ 'what' | get }} you're looking for{{ 'punctuation_mark' | get }}"
}
client = zebr0.Client("http://127.0.0.1:8000", levels=["star_wars", "slang"], configuration_file=Path(""))
assert client.get("template") == "these aren't the droids you're looking for, duh!"
def test_get_latest_image_rolling(server):
server.data = {
"aws-region": region,
"aws-ami-criteria": GET_LATEST_IMAGE_ROLLING_CRITERIA
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
one_week_ago = datetime.utcnow() - timedelta(weeks=1)
creation_date = datetime.fromisoformat(client.get_latest_image().get(
"CreationDate")[:-1]) # removes the "Z" at the end of aws dates
assert creation_date > one_week_ago
def test_get_bucket(server):
server.data = {
"aws-region": region,
"aws-stack-name": stack_name,
"aws-stack-config": GET_BUCKET_CONFIG
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
client.create_stack()
assert client.get_bucket(
) is not None # todo: obviously this is not enough, but it will do for now
def test_create_stack(server):
server.data = {
"aws-region": region,
"aws-stack-name": stack_name,
"aws-stack-config": CREATE_STACK_CONFIG
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
client.create_stack()
boto3_client = boto3.client(service_name="s3", region_name=region)
assert any(bucket
for bucket in boto3_client.list_buckets().get("Buckets", [])
if bucket.get("Name").startswith(stack_name + "-bucket-"))
def run(url: str,
levels: Optional[List[str]],
cache: int,
configuration_file: Path,
reports_path: Path,
key: str,
attempts: int = ATTEMPTS_DEFAULT,
pause: float = PAUSE_DEFAULT,
**_) -> None:
"""
Fetches a script from the key-value server and executes its tasks.
Execution reports are written after each task.
On failure, the output is displayed and the loop stops.
Should you run the script again, successful tasks will be skipped.
:param url: (zebr0) URL of the key-value server, defaults to https://hub.zebr0.io
:param levels: (zebr0) levels of specialization (e.g. ["mattermost", "production"] for a <project>/<environment>/<key> structure), defaults to []
:param cache: (zebr0) in seconds, the duration of the cache of http responses, defaults to 300 seconds
:param configuration_file: (zebr0) path to the configuration file, defaults to /etc/zebr0.conf for a system-wide configuration
:param reports_path: Path to the reports' directory
:param key: the script's key
:param attempts: maximum number of attempts before reporting a failure
:param pause: delay in seconds between two attempts
"""
reports_path.mkdir(parents=True,
exist_ok=True) # make sure the parent directory exists
client = zebr0.Client(url, levels, cache, configuration_file)
for task, status, report_path in recursive_fetch_script(
client, key, reports_path):
if status == Status.SUCCESS:
print("skipping:", json.dumps(task))
continue
print("executing:", json.dumps(task))
report = execute(
**task, attempts=attempts,
pause=pause) if COMMAND in task.keys() else fetch_to_disk(
**task, client=client)
report_path.write_text(json.dumps(report, indent=2),
encoding=zebr0.ENCODING)
if report.get(STATUS) == Status.SUCCESS:
print("success!")
continue
print("error:", json.dumps(report.get(OUTPUT), indent=2))
break
def show(url: str, levels: Optional[List[str]], cache: int,
configuration_file: Path, reports_path: Path, key: str, **_) -> None:
"""
Fetches a script from the key-value server and displays its tasks along with their current status.
:param url: (zebr0) URL of the key-value server, defaults to https://hub.zebr0.io
:param levels: (zebr0) levels of specialization (e.g. ["mattermost", "production"] for a <project>/<environment>/<key> structure), defaults to []
:param cache: (zebr0) in seconds, the duration of the cache of http responses, defaults to 300 seconds
:param configuration_file: (zebr0) path to the configuration file, defaults to /etc/zebr0.conf for a system-wide configuration
:param reports_path: Path to the reports' directory
:param key: the script's key
"""
client = zebr0.Client(url, levels, cache, configuration_file)
for task, status, _ in recursive_fetch_script(client, key, reports_path):
print(f"{status}: {json.dumps(task)}")
def test_create_instance(server):
server.data = {
"aws-region": region,
"aws-stack-name": stack_name,
"aws-stack-config": CREATE_INSTANCE_CONFIG,
"aws-ami-criteria": GET_LATEST_IMAGE_ROLLING_CRITERIA,
"aws-instance-type": "t3.micro",
"aws-volume-size": "8",
"aws-user-data": "#cloud-config"
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
client.create_stack()
assert client.create_instance(
) is not None # todo: obviously this is not enough, but it will do for now
def delete(url: str, levels: Optional[List[str]], cache: int, configuration_file: Path, **_) -> None:
zebr0_client = zebr0.Client(url, levels, cache, configuration_file)
sts_client = sts.Client(zebr0_client)
iam_client = iam.Client(zebr0_client, sts_client)
ec2_client = ec2.Client(zebr0_client, iam_client)
route53_client = route53.Client(zebr0_client)
route53_client.destroy_dns_entry_if_needed()
ec2_client.destroy_address_if_needed()
ec2_client.destroy_instance_if_needed()
iam_client.delete_old_access_keys()
iam_client.delete_user_if_needed()
iam_client.delete_policy_if_needed()
ec2_client.destroy_internet_gateway_if_needed()
ec2_client.destroy_subnet_if_needed()
ec2_client.destroy_vpc_if_needed()
def create(url: str, levels: Optional[List[str]], cache: int, configuration_file: Path, **_) -> None:
zebr0_client = zebr0.Client(url, levels, cache, configuration_file)
sts_client = sts.Client(zebr0_client)
iam_client = iam.Client(zebr0_client, sts_client)
ec2_client = ec2.Client(zebr0_client, iam_client)
route53_client = route53.Client(zebr0_client)
s3_client = s3.Client(zebr0_client)
s3_client.create_bucket_if_needed()
vpc_id = ec2_client.create_vpc_if_needed()
subnet_id = ec2_client.create_subnet_if_needed(vpc_id)
ec2_client.create_internet_gateway_if_needed(vpc_id)
iam_client.create_policy_if_needed()
iam_client.create_user_if_needed()
instance_id = ec2_client.create_instance_if_needed(subnet_id)
address = ec2_client.create_address_if_needed(instance_id)
route53_client.create_dns_entry_if_needed(address)
def test_full(server):
server.data = {
"aws-region": region,
"aws-stack-name": stack_name,
"aws-stack-config": FULL_CONFIG,
"aws-ami-criteria": GET_LATEST_IMAGE_ROLLING_CRITERIA,
"aws-instance-type": "t3.micro",
"aws-volume-size": "8",
"aws-user-data": FULL_USER_DATA
}
client = zebr0_aws.Client(zebr0.Client("http://localhost:8000"))
client.create_stack()
ip = client.create_instance()
heal_check.warning_tolerant_check("http://" + ip + ":2501")
def debug(url: str, levels: Optional[List[str]], cache: int,
configuration_file: Path, reports_path: Path, key: str, **_) -> None:
"""
Fetches a script from the key-value server and executes its tasks through user interaction.
Useful for debugging scripts in a test environment.
:param url: (zebr0) URL of the key-value server, defaults to https://hub.zebr0.io
:param levels: (zebr0) levels of specialization (e.g. ["mattermost", "production"] for a <project>/<environment>/<key> structure), defaults to []
:param cache: (zebr0) in seconds, the duration of the cache of http responses, defaults to 300 seconds
:param configuration_file: (zebr0) path to the configuration file, defaults to /etc/zebr0.conf for a system-wide configuration
:param reports_path: Path to the reports' directory
:param key: the script's key
"""
reports_path.mkdir(parents=True,
exist_ok=True) # make sure the parent directory exists
client = zebr0.Client(url, levels, cache, configuration_file)
for task, status, report_path in recursive_fetch_script(
client, key, reports_path):
if status == Status.SUCCESS:
print("already executed:", json.dumps(task))
print("(s)kip, (e)xecute anyway, or (q)uit?")
else:
print("next:", json.dumps(task))
print("(e)xecute, (s)kip, or (q)uit?")
choice = sys.stdin.readline().strip()
if choice == "e":
report = execute(
**task,
attempts=1) if COMMAND in task.keys() else fetch_to_disk(
**task, client=client)
print("success!" if report.get(STATUS) == Status.SUCCESS else
f"error: {json.dumps(report.get(OUTPUT), indent=2)}")
print("write report? (y)es or (n)o")
choice = sys.stdin.readline().strip()
if choice == "y":
report_path.write_text(json.dumps(report, indent=2),
encoding=zebr0.ENCODING)
elif not choice == "s":
break
def main(args: Optional[List[str]] = None) -> None:
"""
usage: zebr0-lxd [-h] [-u <url>] [-l [<level> [<level> ...]]] [-c <duration>] [-f <path>] [--lxd-url <url>] {create,delete,start,stop} [key]
LXD provisioning based on zebr0 key-value system.
Fetches a stack from the key-value server and manages it on LXD.
positional arguments:
{create,delete,start,stop}
operation to execute on the stack
key the stack's key, defaults to 'lxd-stack'
optional arguments:
-h, --help show this help message and exit
-u <url>, --url <url>
URL of the key-value server, defaults to https://hub.zebr0.io
-l [<level> [<level> ...]], --levels [<level> [<level> ...]]
levels of specialization (e.g. "mattermost production" for a <project>/<environment>/<key> structure), defaults to ""
-c <duration>, --cache <duration>
in seconds, the duration of the cache of http responses, defaults to 300 seconds
-f <path>, --configuration-file <path>
path to the configuration file, defaults to /etc/zebr0.conf for a system-wide configuration
--lxd-url <url> URL of the LXD API (scheme is "http+unix", socket path is percent-encoded into the host field), defaults to "http+unix://%2Fvar%2Fsnap%2Flxd%2Fcommon%2Flxd%2Funix.socket"
"""
argparser = zebr0.build_argument_parser(description="LXD provisioning based on zebr0 key-value system.\nFetches a stack from the key-value server and manages it on LXD.", formatter_class=argparse.RawDescriptionHelpFormatter)
argparser.add_argument("command", choices=["create", "delete", "start", "stop"], help="operation to execute on the stack")
argparser.add_argument("key", nargs="?", default="lxd-stack", help="the stack's key, defaults to 'lxd-stack'")
argparser.add_argument("--lxd-url", default=URL_DEFAULT, help='URL of the LXD API (scheme is "http+unix", socket path is percent-encoded into the host field), defaults to "http+unix://%%2Fvar%%2Fsnap%%2Flxd%%2Fcommon%%2Flxd%%2Funix.socket"', metavar="<url>")
args = argparser.parse_args(args)
value = zebr0.Client(args.url, args.levels, args.cache, args.configuration_file).get(args.key)
if not value:
print(f"key '{args.key}' not found on server {args.url}")
exit(1)
stack = yaml.load(value, Loader=yaml.BaseLoader)
if not isinstance(stack, dict):
print(f"key '{args.key}' on server {args.url} is not a proper yaml or json dictionary")
exit(1)
getattr(Client(args.lxd_url), args.command + "_stack")(stack)
def test_ok(server, tmp_path, capsys, monkeypatch):
server.data = {
"script":
["echo one", {
"command": "sleep 1 && echo two",
"variant": "omicron"
}]
}
configuration_file = tmp_path.joinpath("zebr0.conf")
zebr0.Client("http://localhost:8000", ["lorem", "ipsum"], 1,
configuration_file).save_configuration()
reports_path = tmp_path.joinpath("reports")
zebr0_script.main(f"-r {reports_path} log".split())
assert capsys.readouterr().out == ""
zebr0_script.main(
f"-f {configuration_file} -r {reports_path} show".split())
assert capsys.readouterr().out == OK_OUTPUT1
monkeypatch.setattr("sys.stdin", io.StringIO("e\nn\nq\n"))
zebr0_script.main(
f"-f {configuration_file} -r {reports_path} debug".split())
assert capsys.readouterr().out == OK_OUTPUT2
zebr0_script.main(f"-f {configuration_file} -r {reports_path} run".split())
assert capsys.readouterr().out == OK_OUTPUT3
zebr0_script.main(f"-r {reports_path} log".split())
assert capsys.readouterr().out == OK_OUTPUT4.format(
format_mtime(
reports_path.joinpath("eb1cf90440f34ae3823016216940e5c2")),
format_mtime(
reports_path.joinpath("e3c088ff80db25e139905e254860b0e3")))
zebr0_script.main(
f"-f {configuration_file} -r {reports_path} show".split())
assert capsys.readouterr().out == OK_OUTPUT5
zebr0_script.main(f"-f {configuration_file} -r {reports_path} run".split())
assert capsys.readouterr().out == OK_OUTPUT6
def test_cache(server):
server.access_logs = [] # resetting server logs from previous tests
server.data = {"ping": "pong", "yin": "yang"}
client = zebr0.Client("http://127.0.0.1:8000", cache=1, configuration_file=Path("")) # cache of 1 second for the purposes of the test
assert client.get("ping") == "pong" # "pong" is now in cache for "/ping"
time.sleep(0.5)
assert client.get("yin") == "yang" # "yang" is now in cache for "/yin"
time.sleep(0.1)
server.data = {"ping": "peng", "yin": "yeng"} # new values, shouldn't be used until cache has expired
time.sleep(0.3)
assert client.get("ping") == "pong" # using cache for "/ping"
time.sleep(0.2)
assert client.get("ping") == "peng" # 1.1 second has passed, cache has expired for "/ping", now fetching the new value
assert client.get("yin") == "yang" # still using cache for "/yin"
time.sleep(0.5)
assert client.get("yin") == "yeng" # cache also has expired for "/yin", now fetching the new value
assert server.access_logs == ["/ping", "/yin", "/ping", "/yin"]
def test_default_levels(server):
server.data = {"knock-knock": "who's there?"}
client = zebr0.Client("http://127.0.0.1:8000", configuration_file=Path(""))
assert client.get("knock-knock") == "who's there?"
def test_deepest_level(server):
server.data = {"lorem/ipsum/dolor": "sit amet"}
client = zebr0.Client("http://127.0.0.1:8000", levels=["lorem", "ipsum"], configuration_file=Path(""))
assert client.get("dolor") == "sit amet"
def test_intermediate_level(server):
server.data = {"consectetur/elit": "sed do"}
client = zebr0.Client("http://127.0.0.1:8000", levels=["consectetur", "adipiscing"], configuration_file=Path(""))
assert client.get("elit") == "sed do"
def test_root_level(server):
server.data = {"incididunt": "ut labore"}
client = zebr0.Client("http://127.0.0.1:8000", levels=["eiusmod", "tempor"], configuration_file=Path(""))
assert client.get("incididunt") == "ut labore"
def test_missing_key_and_default_value(server):
server.data = {}
client = zebr0.Client("http://127.0.0.1:8000", levels=["dolore", "magna"], configuration_file=Path(""))
assert client.get("aliqua") == ""
assert client.get("aliqua", default="default") == "default"
def test_strip(server):
server.data = {"knock-knock": "\nwho's there?\n"}
client = zebr0.Client("http://127.0.0.1:8000", configuration_file=Path(""))
assert client.get("knock-knock", strip=False) == "\nwho's there?\n"
assert client.get("knock-knock") == "who's there?"
def test_basic_render(server):
server.data = {"template": "{{ url }} {{ levels[0] }} {{ levels[1] }}"}
client = zebr0.Client("http://127.0.0.1:8000", levels=["lorem", "ipsum"], configuration_file=Path(""))
assert client.get("template", template=False) == "{{ url }} {{ levels[0] }} {{ levels[1] }}"
assert client.get("template") == "http://127.0.0.1:8000 lorem ipsum"
def client():
return zebr0.Client("http://localhost:8000", configuration_file=Path(""))
def test_render_with_default(server):
server.data = {"template": "{{ 'missing_key' | get('default') }}"}
client = zebr0.Client("http://127.0.0.1:8000", configuration_file=Path(""))
assert client.get("template") == "default"
def test_default_url():
client = zebr0.Client(configuration_file=Path(""))
assert client.get("domain-name") == "zebr0.io"
