def stop_schedule(cluster_name: str, schedule_name: str, **kwargs): # Load details cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.stop_schedule(schedule_name=schedule_name)
def start_schedule(cluster_name: str, deployment_path: str, **kwargs): # Load details cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.start_schedule(deployment_path=deployment_path)
def list_job(cluster_name: str, **kwargs): # Load details cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.list_job()
def get_job_logs(cluster_name: str, job_name: str, **kwargs): # Load details cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.get_job_logs(job_name=job_name)
def setUpClass(cls, file_path: str = os.path.abspath(__file__)) -> None: # Get and set params GlobalParams.LOG_LEVEL = logging.DEBUG cls.test_id = uuid.uuid4().hex[:8] os.makedirs( os.path.expanduser(f"{GlobalPaths.MARO_TEST}/{cls.test_id}"), exist_ok=True) os.makedirs( os.path.expanduser(f"{GlobalPaths.MARO_TEST}/{cls.test_id}/tar"), exist_ok=True) cls.file_path = os.path.abspath(__file__) cls.dir_path = os.path.dirname(cls.file_path) cls.deployment_template_path = os.path.normpath( os.path.join(cls.dir_path, "../templates/test_k8s_azure_create.yml")) cls.deployment_path = os.path.expanduser( f"{GlobalPaths.MARO_TEST}/{cls.test_id}/test_k8s_azure_create.yml") cls.config_path = os.path.normpath( os.path.join(cls.dir_path, "../config.yml")) # Load config and save deployment with open(cls.deployment_template_path) as fr: deployment_details = yaml.safe_load(fr) with open(cls.config_path) as fr: config_details = yaml.safe_load(fr) if config_details["cloud/subscription"] and config_details[ "user/admin_public_key"]: deployment_details["cloud"]["subscription"] = config_details[ "cloud/subscription"] deployment_details["user"][ "admin_public_key"] = config_details[ "user/admin_public_key"] else: raise Exception("Invalid config") with open(cls.deployment_path, "w") as fw: yaml.safe_dump(deployment_details, fw) # Get params from deployments cls.cluster_name = deployment_details["name"] # Init test files cls.local_big_file_path = os.path.expanduser( f"{GlobalPaths.MARO_TEST}/{cls.test_id}/big_file") cls.local_small_files_path = os.path.expanduser( f"{GlobalPaths.MARO_TEST}/{cls.test_id}/small_files") command = f"dd if=/dev/zero of={cls.local_big_file_path} bs=1 count=0 seek=1G" SubProcess.run(command) command = f"git clone [email protected]:microsoft/maro.git {cls.local_small_files_path}" SubProcess.run(command) # Create cluster command = f"maro k8s create --debug {cls.deployment_path}" SubProcess.interactive_run(command) cls.cluster_details = load_cluster_details( cluster_name=cls.cluster_name) cls.cluster_id = cls.cluster_details["id"] cls.executor = K8sAzureExecutor(cluster_name=cls.cluster_name) time.sleep(15) cls.pod_name = cls._get_redis_pod_name()
def create(deployment_path: str, **kwargs): with open(deployment_path, 'r') as fr: create_deployment = yaml.safe_load(fr) try: if create_deployment['mode'] == 'k8s': if create_deployment['cloud']['infra'] == 'azure': K8sAzureExecutor.build_cluster_details( create_deployment=create_deployment) executor = K8sAzureExecutor( cluster_name=create_deployment['name']) executor.create() else: raise ParsingError( f"Deployment is broken: Invalid infra: {create_deployment['cloud']['infra']}" ) else: raise ParsingError( f"Deployment is broken: Invalid mode: {create_deployment['mode']}" ) except KeyError as e: raise ParsingError(f"Deployment is broken: Missing key: '{e.args[0]}'")
def scale_node(cluster_name: str, replicas: int, node_size: str, **kwargs): cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.scale_node(replicas=replicas, node_size=node_size)
def remove_data(cluster_name: str, remote_path: str, **kwargs): cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.remove_data(remote_path=remote_path)
def pull_data(cluster_name: str, local_dir: str, remote_path: str, **kwargs): cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.pull_data(local_dir=local_dir, remote_path=remote_path)
def status(cluster_name: str, **kwargs): cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.status()
def template(export_path: str, **kwargs): K8sAzureExecutor.template(export_path=export_path)
def push_image(cluster_name: str, image_name: str, **kwargs): cluster_details = load_cluster_details(cluster_name=cluster_name) if cluster_details['cloud']['infra'] == 'azure': executor = K8sAzureExecutor(cluster_name=cluster_name) executor.push_image(image_name=image_name)