def push_data(cluster_name: str, local_path: str, remote_path: str, **kwargs):
# Load details
cluster_details = load_cluster_details(cluster_name=cluster_name)
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master']['public_ip_address']
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/data/{remote_path}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
copy_files_to_node(
local_path=local_path,
remote_dir=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/data/{remote_path}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
def _sync_job_details(cluster_name: str, job_name: str,
job_details: dict) -> None:
try:
cluster_details = load_cluster_details(cluster_name=cluster_name)
admin_username = cluster_details["user"]["admin_username"]
master_public_ip_address = cluster_details["master"][
"public_ip_address"]
ssh_port = cluster_details["connection"]["ssh"]["port"]
except Exception:
return
copy_files_to_node(
local_path=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/jobs/{job_name}/details.yml",
remote_dir=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/jobs/{job_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address,
ssh_port=ssh_port)
logger.debug("Sync job details with master.")
def _start_job(self, job_details: dict):
logger.info(f"Start sending job ticket {job_details['name']}")
# Load details
cluster_details = self.cluster_details
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master'][
'public_ip_address']
job_name = job_details['name']
# Sync mkdir
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Save job deployment
save_job_details(cluster_name=self.cluster_name,
job_name=job_name,
job_details=job_details)
# Set job id
self._set_job_id(job_name=job_name)
# Sync job details to master
copy_files_to_node(
local_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/details.yml",
remote_dir=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Remote start job
self.grass_executor.remote_create_job_details(job_name=job_name)
self.grass_executor.remote_create_pending_job_ticket(job_name=job_name)
logger.info_green(f"Job ticket {job_details['name']} is sent")
def _init_master(self):
logger.info(f"Initializing master node")
# Load details
cluster_details = self.cluster_details
master_details = cluster_details['master']
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master'][
'public_ip_address']
# Make sure master is able to connect
self.grass_executor.retry_until_connected(
node_ip_address=master_public_ip_address)
# Create folders
sync_mkdir(remote_path=GlobalPaths.MARO_GRASS_LIB,
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/data",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/schedules",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Copy required files
copy_files_to_node(local_path=f"{GlobalPaths.MARO_GRASS_LIB}/*",
remote_dir=GlobalPaths.MARO_GRASS_LIB,
admin_username=admin_username,
node_ip_address=master_public_ip_address)
copy_files_to_node(
local_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/*",
remote_dir=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Get public key
public_key = self.grass_executor.remote_get_public_key(
node_ip_address=master_public_ip_address)
# Remote init master
self.grass_executor.remote_init_master()
# Load master agent service
self.grass_executor.remote_load_master_agent_service()
# Save details
master_details['public_key'] = public_key
master_details['image_files'] = {}
save_cluster_details(cluster_name=self.cluster_name,
cluster_details=cluster_details)
self.grass_executor.remote_set_master_details(
master_details=cluster_details['master'])
logger.info_green(f"Master node is initialized")
def _init_node(self, node_name: str):
logger.info(f"Initiating node {node_name}.")
# Load details
cluster_details = self.cluster_details
admin_username = cluster_details["user"]["admin_username"]
node_details = self.grass_executor.remote_get_node_details(
node_name=node_name)
node_public_ip_address = node_details["public_ip_address"]
ssh_port = cluster_details["connection"]["ssh"]["port"]
# Make sure the node is able to connect
self.grass_executor.retry_connection_and_set_ssh_port(
node_ip_address=node_public_ip_address)
# Create folders
path_list = {
GlobalPaths.MARO_GRASS_LIB,
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/data",
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images",
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs",
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/schedules"
}
self._create_path_in_list(node_public_ip_address, path_list)
# Copy required files
copy_files_to_node(local_path=GlobalPaths.MARO_GRASS_LIB,
remote_dir=GlobalPaths.MARO_LIB,
admin_username=admin_username,
node_ip_address=node_public_ip_address,
ssh_port=ssh_port)
copy_files_to_node(
local_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
remote_dir=GlobalPaths.MARO_CLUSTERS,
admin_username=admin_username,
node_ip_address=node_public_ip_address,
ssh_port=ssh_port)
# Remote init node
self.grass_executor.remote_init_node(
node_name=node_name, node_ip_address=node_public_ip_address)
# Get public key
public_key = self.grass_executor.remote_get_public_key(
node_ip_address=node_public_ip_address)
# Save details
node_details["public_key"] = public_key
self.grass_executor.remote_set_node_details(node_name=node_name,
node_details=node_details)
# Update node status
# Since On-Premises machines don't need to shutdown, it will be set to start directly.
self.grass_executor.remote_update_node_status(node_name=node_name,
action="start")
# Load images
self.grass_executor.remote_load_images(
node_name=node_name,
parallels=GlobalParams.PARALLELS,
node_ip_address=node_public_ip_address)
# Load node agent service
self.grass_executor.remote_load_node_agent_service(
node_name=node_name, node_ip_address=node_public_ip_address)
logger.info_green(f"Node {node_name} has been initialized.")
def push_image(self, image_name: str, image_path: str, remote_context_path: str,
remote_image_name: str):
# Load details
cluster_details = self.cluster_details
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master']['public_ip_address']
# Get images dir
images_dir = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images"
# Push image
if image_name:
new_file_name = get_valid_file_name(image_name)
image_path = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/{new_file_name}"
self._save_image(
image_name=image_name,
export_path=os.path.expanduser(image_path)
)
if self._check_checksum_validity(
local_file_path=os.path.expanduser(image_path),
remote_file_path=os.path.join(images_dir, image_name)
):
logger.info_green(f"The image file '{new_file_name}' already exists")
return
copy_files_to_node(
local_path=image_path,
remote_dir=images_dir,
admin_username=admin_username, node_ip_address=master_public_ip_address
)
self.grass_executor.remote_update_image_files_details()
self._batch_load_images()
logger.info_green(f"Image {image_name} is loaded")
elif image_path:
file_name = os.path.basename(image_path)
new_file_name = get_valid_file_name(file_name)
image_path = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/{new_file_name}"
copy_and_rename(
source_path=os.path.expanduser(image_path),
target_dir=image_path
)
if self._check_checksum_validity(
local_file_path=os.path.expanduser(image_path),
remote_file_path=os.path.join(images_dir, new_file_name)
):
logger.info_green(f"The image file '{new_file_name}' already exists")
return
copy_files_to_node(
local_path=image_path,
remote_dir=images_dir,
admin_username=admin_username, node_ip_address=master_public_ip_address
)
self.grass_executor.remote_update_image_files_details()
self._batch_load_images()
elif remote_context_path and remote_image_name:
self.grass_executor.remote_build_image(
remote_context_path=remote_context_path,
remote_image_name=remote_image_name
)
self._batch_load_images()
else:
raise CliException("Invalid arguments")