def __init__(self, cluster_details: dict):
self.cluster_details = cluster_details
# Upper configs
self.cluster_name = self.cluster_details["name"]
self.cluster_id = self.cluster_details["id"]
# User configs
self.user_details = DetailsReader.load_default_user_details(
cluster_name=self.cluster_name)
# Master configs
self.master_username = self.cluster_details["master"]["username"]
self.master_public_ip_address = self.cluster_details["master"][
"public_ip_address"]
self.master_private_ip_address = self.cluster_details["master"][
"private_ip_address"]
self.master_redis_port = self.cluster_details["master"]["redis"][
"port"]
self.master_hostname = self.cluster_details["master"]["hostname"]
self.master_api_client = MasterApiClientV1(
master_hostname=self.master_public_ip_address,
master_api_server_port=self.cluster_details["master"]["api_server"]
["port"],
user_id=self.user_details["id"],
master_to_dev_encryption_private_key=self.
user_details["master_to_dev_encryption_private_key"],
dev_to_master_encryption_public_key=self.
user_details["dev_to_master_encryption_public_key"],
dev_to_master_signing_private_key=self.
user_details["dev_to_master_signing_private_key"])
self.master_ssh_port = self.cluster_details["master"]["ssh"]["port"]
self.master_api_server_port = self.cluster_details["master"][
"api_server"]["port"]
def setUpClass(cls, file_path: str = os.path.abspath(__file__)) -> None:
# Get and set params
GlobalParams.LOG_LEVEL = logging.DEBUG
cls.test_id = uuid.uuid4().hex[:8]
os.makedirs(
os.path.expanduser(f"{GlobalPaths.MARO_TEST}/{cls.test_id}"),
exist_ok=True)
os.makedirs(
os.path.expanduser(f"{GlobalPaths.MARO_TEST}/{cls.test_id}/tar"),
exist_ok=True)
cls.file_path = os.path.abspath(__file__)
cls.dir_path = os.path.dirname(cls.file_path)
cls.deployment_template_path = os.path.normpath(
os.path.join(cls.dir_path,
"../templates/test_k8s_azure_create.yml"))
cls.deployment_path = os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{cls.test_id}/test_k8s_azure_create.yml")
cls.config_path = os.path.normpath(
os.path.join(cls.dir_path, "../config.yml"))
# Load config and save deployment
with open(cls.deployment_template_path) as fr:
deployment_details = yaml.safe_load(fr)
with open(cls.config_path) as fr:
config_details = yaml.safe_load(fr)
if config_details["cloud/subscription"] and config_details[
"user/admin_public_key"]:
deployment_details["cloud"]["subscription"] = config_details[
"cloud/subscription"]
deployment_details["user"][
"admin_public_key"] = config_details[
"user/admin_public_key"]
else:
raise Exception("Invalid config")
with open(cls.deployment_path, "w") as fw:
yaml.safe_dump(deployment_details, fw)
# Get params from deployments
cls.cluster_name = deployment_details["name"]
# Init test files
cls.local_big_file_path = os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{cls.test_id}/big_file")
cls.local_small_files_path = os.path.expanduser(
f"{GlobalPaths.MARO_TEST}/{cls.test_id}/small_files")
command = f"dd if=/dev/zero of={cls.local_big_file_path} bs=1 count=0 seek=1G"
Subprocess.run(command=command)
command = f"git clone [email protected]:microsoft/maro.git {cls.local_small_files_path}"
Subprocess.run(command=command)
# Create cluster
command = f"maro k8s create --debug {cls.deployment_path}"
Subprocess.interactive_run(command=command)
cls.cluster_details = DetailsReader.load_cluster_details(
cluster_name=cls.cluster_name)
cls.cluster_id = cls.cluster_details["id"]
cls.executor = K8sAksExecutor(cluster_name=cls.cluster_name)
time.sleep(15)
cls.pod_name = cls._get_redis_pod_name()
def __init__(self):
self.cluster_detail = DetailsReader.load_cluster_details("process")
self.check_interval = self.cluster_detail["check_interval"]
self.redis_connection = redis.Redis(
host=self.cluster_detail["redis_info"]["host"],
port=self.cluster_detail["redis_info"]["port"]
)
self.redis_connection.hset(ProcessRedisName.SETTING, "agent_pid", os.getpid())
def __init__(self, cluster_name: str):
self.cluster_name = cluster_name
self.cluster_detail = DetailsReader.load_cluster_details(cluster_name)
self.check_interval = self.cluster_detail["master"]["agents"]["check_interval"]
self.redis_connection = redis.Redis(
host="localhost",
port=self.cluster_detail["master"]["redis"]["port"]
)
self.redis_connection.hset(f"{self.cluster_name}:runtime_detail", "agent_id", os.getpid())
def __init__(self, cluster_name: str):
self.cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
# Cloud configs
self.subscription = self.cluster_details["cloud"]["subscription"]
self.resource_group = self.cluster_details["cloud"]["resource_group"]
self.location = self.cluster_details["cloud"]["location"]
super().__init__(cluster_details=self.cluster_details)
def delete(cluster_name: str, **kwargs):
# Late import.
from maro.cli.k8s.executors.k8s_aks_executor import K8sAksExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
cluster_details = DetailsReader.load_cluster_details(cluster_name=cluster_name)
if cluster_details["mode"] == "k8s/aks":
executor = K8sAksExecutor(cluster_name=cluster_name)
executor.delete()
else:
raise BadRequestError(f"Unsupported operation in mode '{cluster_details['mode']}'.")
def clean(cluster_name: str, **kwargs):
# Late import.
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
if cluster_details["mode"] == "grass/azure":
executor = GrassAzureExecutor(cluster_name=cluster_name)
executor.clean()
else:
raise BadRequestError(
f"Unsupported operation in mode '{cluster_details['mode']}'.")
def __init__(self, cluster_name: str):
super().__init__(cluster_details=DetailsReader.load_cluster_details(
cluster_name=cluster_name))
# Cloud configs
self.subscription = self.cluster_details["cloud"]["subscription"]
self.resource_group = self.cluster_details["cloud"]["resource_group"]
self.location = self.cluster_details["cloud"]["location"]
self.default_username = self.cluster_details["cloud"][
"default_username"]
# Connection configs
self.ssh_port = self.cluster_details["connection"]["ssh"]["port"]
self.api_server_port = self.cluster_details["connection"][
"api_server"]["port"]
def start_schedule(cluster_name: str, deployment_path: str, **kwargs):
# Late import.
from maro.cli.k8s.executors.k8s_aks_executor import K8sAksExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
# Load details
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
if cluster_details["mode"] == "k8s/aks":
executor = K8sAksExecutor(cluster_name=cluster_name)
executor.start_schedule(deployment_path=deployment_path)
else:
raise BadRequestError(
f"Unsupported operation in mode '{cluster_details['mode']}'.")
def pull_data(cluster_name: str, local_path: str, remote_path: str, **kwargs):
# Late import.
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
cluster_details = DetailsReader.load_cluster_details(cluster_name=cluster_name)
if cluster_details["mode"] == "grass/azure":
executor = GrassAzureExecutor(cluster_name=cluster_name)
executor.pull_data(local_path=local_path, remote_path=remote_path)
elif cluster_details["mode"] == "grass/on-premises":
executor = GrassOnPremisesExecutor(cluster_name=cluster_name)
executor.pull_data(local_path=local_path, remote_path=remote_path)
else:
raise BadRequestError(f"Unsupported operation in mode '{cluster_details['mode']}'.")
def __init__(self, cluster_name: str, cluster_details: dict = None):
self.cluster_name = cluster_name
self.cluster_details = DetailsReader.load_cluster_details(cluster_name=cluster_name) \
if not cluster_details else cluster_details
# Connection with Redis
redis_port = self.cluster_details["master"]["redis"]["port"]
self._redis_connection = redis.Redis(host="localhost", port=redis_port)
try:
self._redis_connection.ping()
except Exception:
redis_process = subprocess.Popen(
["redis-server", "--port", str(redis_port), "--daemonize yes"]
)
redis_process.wait(timeout=2)
# Connection with Resource Redis
self._resource_redis = LocalResourceExecutor()
def load_executor(cluster_name):
if cluster_name == "process":
from maro.cli.process.executor import ProcessExecutor
executor = ProcessExecutor()
cluster_type = DashboardType.PROCESS
else:
from maro.cli.utils.details_reader import DetailsReader
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
if cluster_details["mode"] == "grass/azure":
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
executor = GrassAzureExecutor(cluster_name=cluster_name)
cluster_type = DashboardType.AZURE
elif cluster_details["mode"] == "grass/on-premises":
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
executor = GrassOnPremisesExecutor(cluster_name=cluster_name)
cluster_type = DashboardType.ONPREMISES
elif cluster_details["mode"] == "grass/local":
from maro.cli.grass.executors.grass_local_executor import GrassLocalExecutor
executor = GrassLocalExecutor(cluster_name=cluster_name)
cluster_type = DashboardType.LOCAL
return executor, cluster_type
def get_job_logs(cluster_name: str, job_name: str, **kwargs):
# Late import.
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
from maro.cli.grass.executors.grass_local_executor import GrassLocalExecutor
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
if cluster_details["mode"] == "grass/azure":
executor = GrassAzureExecutor(cluster_name=cluster_name)
executor.get_job_logs(job_name=job_name)
elif cluster_details["mode"] == "grass/local":
executor = GrassLocalExecutor(cluster_name=cluster_name)
executor.get_job_logs(job_name=job_name)
elif cluster_details["mode"] == "grass/on-premises":
executor = GrassOnPremisesExecutor(cluster_name=cluster_name)
executor.get_job_logs(job_name=job_name)
else:
raise BadRequestError(
f"Unsupported operation in mode '{cluster_details['mode']}'.")
def start_schedule(cluster_name: str, deployment_path: str, **kwargs):
# Late import.
from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
from maro.cli.grass.executors.grass_local_executor import GrassLocalExecutor
from maro.cli.grass.executors.grass_on_premises_executor import GrassOnPremisesExecutor
from maro.cli.utils.details_reader import DetailsReader
from maro.utils.exception.cli_exception import BadRequestError
# Load details
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
if cluster_details["mode"] == "grass/azure":
executor = GrassAzureExecutor(cluster_name=cluster_name)
executor.start_schedule(deployment_path=deployment_path)
elif cluster_details["mode"] == "grass/local":
executor = GrassLocalExecutor(cluster_name=cluster_name)
executor.start_schedule(deployment_path=deployment_path)
elif cluster_details["mode"] == "grass/on-premises":
executor = GrassOnPremisesExecutor(cluster_name=cluster_name)
executor.start_schedule(deployment_path=deployment_path)
else:
raise BadRequestError(
f"Unsupported operation in mode '{cluster_details['mode']}'.")
def with_checker(*args, **kwargs):
# Get params
cluster_name = kwargs["cluster_name"]
# Get details
try:
cluster_details = DetailsReader.load_cluster_details(
cluster_name=cluster_name)
# Check details validity
if cluster_details["mode"] not in {
"grass/azure", "k8s/aks", "grass/on-premises",
"grass/local"
}:
raise ClusterInternalError(
f"Cluster details are broken: Invalid mode '{cluster_details['mode']}'."
)
except FileNotFoundError:
raise BadRequestError(f"Cluster '{cluster_name}' is not found.")
except KeyError:
raise ClusterInternalError(
"Cluster details are broken: Missing key 'mode'.")
func(*args, **kwargs)
def __init__(self, cluster_name: str):
super().__init__(cluster_details=DetailsReader.load_cluster_details(
cluster_name=cluster_name))