def start_schedule(self, deployment_path: str):
# Load start_schedule_deployment
with open(deployment_path, 'r') as fr:
start_schedule_deployment = yaml.safe_load(fr)
# Standardize start_schedule_deployment
self._standardize_start_schedule_deployment(
start_schedule_deployment=start_schedule_deployment)
schedule_name = start_schedule_deployment['name']
# Load details
cluster_details = self.cluster_details
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master'][
'public_ip_address']
# Sync mkdir
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/schedules/{schedule_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Save schedule deployment
save_schedule_details(cluster_name=self.cluster_name,
schedule_name=schedule_name,
schedule_details=start_schedule_deployment)
# Start jobs
for job_name in start_schedule_deployment['job_names']:
job_details = self._build_job_details(
schedule_details=start_schedule_deployment, job_name=job_name)
self._start_job(job_details=job_details)
def push_data(cluster_name: str, local_path: str, remote_path: str, **kwargs):
# Load details
cluster_details = load_cluster_details(cluster_name=cluster_name)
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master']['public_ip_address']
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/data/{remote_path}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
copy_files_to_node(
local_path=local_path,
remote_dir=
f"{GlobalPaths.MARO_CLUSTERS}/{cluster_name}/data/{remote_path}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
def _start_job(self, job_details: dict):
logger.info(f"Start sending job ticket {job_details['name']}")
# Load details
cluster_details = self.cluster_details
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master'][
'public_ip_address']
job_name = job_details['name']
# Sync mkdir
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Save job deployment
save_job_details(cluster_name=self.cluster_name,
job_name=job_name,
job_details=job_details)
# Set job id
self._set_job_id(job_name=job_name)
# Sync job details to master
copy_files_to_node(
local_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}/details.yml",
remote_dir=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs/{job_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Remote start job
self.grass_executor.remote_create_job_details(job_name=job_name)
self.grass_executor.remote_create_pending_job_ticket(job_name=job_name)
logger.info_green(f"Job ticket {job_details['name']} is sent")
def _init_master(self):
logger.info(f"Initializing master node")
# Load details
cluster_details = self.cluster_details
master_details = cluster_details['master']
admin_username = cluster_details['user']['admin_username']
master_public_ip_address = cluster_details['master'][
'public_ip_address']
# Make sure master is able to connect
self.grass_executor.retry_until_connected(
node_ip_address=master_public_ip_address)
# Create folders
sync_mkdir(remote_path=GlobalPaths.MARO_GRASS_LIB,
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/data",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/jobs",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
sync_mkdir(
remote_path=
f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/schedules",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Copy required files
copy_files_to_node(local_path=f"{GlobalPaths.MARO_GRASS_LIB}/*",
remote_dir=GlobalPaths.MARO_GRASS_LIB,
admin_username=admin_username,
node_ip_address=master_public_ip_address)
copy_files_to_node(
local_path=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/*",
remote_dir=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}",
admin_username=admin_username,
node_ip_address=master_public_ip_address)
# Get public key
public_key = self.grass_executor.remote_get_public_key(
node_ip_address=master_public_ip_address)
# Remote init master
self.grass_executor.remote_init_master()
# Load master agent service
self.grass_executor.remote_load_master_agent_service()
# Save details
master_details['public_key'] = public_key
master_details['image_files'] = {}
save_cluster_details(cluster_name=self.cluster_name,
cluster_details=cluster_details)
self.grass_executor.remote_set_master_details(
master_details=cluster_details['master'])
logger.info_green(f"Master node is initialized")