def list_jobs(self): try: return [ models.Job(cloud_job_schedule) for cloud_job_schedule in job_submit_helper.list_jobs(self) ] except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def list_jobs(core_job_operations): try: return [ models.Job(cloud_job_schedule) for cloud_job_schedule in _list_jobs(core_job_operations) ] except BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def submit_job(self, job_configuration): try: cluster_data = self._get_cluster_data(job_configuration.id) node_data = NodeData( job_configuration.as_cluster_config()).add_core().done() zip_resource_files = cluster_data.upload_node_data( node_data).to_resource_file() start_task = create_cluster_helper.generate_cluster_start_task( self, zip_resource_files, job_configuration.gpu_enabled, job_configuration.docker_repo, worker_on_master=job_configuration.worker_on_master) application_tasks = [] for application in job_configuration.applications: application_tasks.append((application, cluster_submit_helper.generate_task( self, job_configuration.id, application))) job_manager_task = job_submit_helper.generate_task( self, job_configuration, application_tasks) software_metadata_key = "spark" vm_image = models.VmImage(publisher='Canonical', offer='UbuntuServer', sku='16.04') if job_configuration.max_dedicated_nodes and not job_configuration.max_low_pri_nodes: autoscale_formula = "maxNumberofVMs = {0}; targetNumberofVMs = {1};" \ " $TargetDedicatedNodes=min(maxNumberofVMs, targetNumberofVMs)".format( job_configuration.max_dedicated_nodes, job_configuration.max_dedicated_nodes) elif job_configuration.max_low_pri_nodes and not job_configuration.max_dedicated_nodes: autoscale_formula = "maxNumberofVMs = {0}; targetNumberofVMs = {1};" \ " $TargetLowPriorityNodes=min(maxNumberofVMs, targetNumberofVMs)".format( job_configuration.max_low_pri_nodes, job_configuration.max_low_pri_nodes) else: raise error.AztkError("Jobs do not support both dedicated and low priority nodes." \ " JobConfiguration fields max_dedicated_nodes and max_low_pri_nodes are mutually exclusive values.") job = self.__submit_job( job_configuration=job_configuration, start_task=start_task, job_manager_task=job_manager_task, autoscale_formula=autoscale_formula, software_metadata_key=software_metadata_key, vm_image_model=vm_image, application_metadata='\n'.join( application.name for application in (job_configuration.applications or []))) return models.Job(job) except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def submit_job(self, job_configuration: models.JobConfiguration): try: job_configuration = _apply_default_for_job_config( job_configuration) job_configuration.validate() cluster_data = self._get_cluster_data(job_configuration.id) node_data = NodeData( job_configuration.to_cluster_config()).add_core().done() zip_resource_files = cluster_data.upload_node_data( node_data).to_resource_file() start_task = create_cluster_helper.generate_cluster_start_task( self, zip_resource_files, job_configuration.id, job_configuration.gpu_enabled, job_configuration.get_docker_repo(), mixed_mode=job_configuration.mixed_mode(), worker_on_master=job_configuration.worker_on_master) application_tasks = [] for application in job_configuration.applications: application_tasks.append((application, cluster_submit_helper.generate_task( self, job_configuration.id, application))) job_manager_task = job_submit_helper.generate_task( self, job_configuration, application_tasks) software_metadata_key = "spark" vm_image = models.VmImage(publisher='Canonical', offer='UbuntuServer', sku='16.04') autoscale_formula = "$TargetDedicatedNodes = {0}; " \ "$TargetLowPriorityNodes = {1}".format( job_configuration.max_dedicated_nodes, job_configuration.max_low_pri_nodes) job = self.__submit_job( job_configuration=job_configuration, start_task=start_task, job_manager_task=job_manager_task, autoscale_formula=autoscale_formula, software_metadata_key=software_metadata_key, vm_image_model=vm_image, application_metadata='\n'.join( application.name for application in (job_configuration.applications or []))) return models.Job(job) except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def submit_job(core_job_operations, spark_job_operations, job_configuration: models.JobConfiguration, wait: bool = False): try: job_configuration = _apply_default_for_job_config(job_configuration) job_configuration.validate() cluster_data = core_job_operations.get_cluster_data(job_configuration.id) node_data = NodeData(job_configuration.to_cluster_config()).add_core().done() zip_resource_files = cluster_data.upload_node_data(node_data).to_resource_file() start_task = spark_job_operations._generate_cluster_start_task( core_job_operations, zip_resource_files, job_configuration.id, job_configuration.gpu_enabled, job_configuration.get_docker_repo(), job_configuration.get_docker_run_options(), mixed_mode=job_configuration.mixed_mode(), worker_on_master=job_configuration.worker_on_master, ) application_tasks = [] for application in job_configuration.applications: application_tasks.append(( application, spark_job_operations._generate_application_task(core_job_operations, job_configuration.id, application), )) job_manager_task = generate_job_manager_task(core_job_operations, job_configuration, application_tasks) software_metadata_key = base_models.Software.spark vm_image = models.VmImage(publisher="Canonical", offer="UbuntuServer", sku="16.04") autoscale_formula = "$TargetDedicatedNodes = {0}; " "$TargetLowPriorityNodes = {1}".format( job_configuration.max_dedicated_nodes, job_configuration.max_low_pri_nodes) job = core_job_operations.submit( job_configuration=job_configuration, start_task=start_task, job_manager_task=job_manager_task, autoscale_formula=autoscale_formula, software_metadata_key=software_metadata_key, vm_image_model=vm_image, application_metadata="\n".join(application.name for application in (job_configuration.applications or [])), ) if wait: spark_job_operations.wait(id=job_configuration.id) return models.Job(job) except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def get_job(self, job_id): try: job, apps, pool, nodes = job_submit_helper.get_job(self, job_id) return models.Job(job, apps, pool, nodes) except batch_error.BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))
def get_job(core_job_operations, job_id): try: job, tasks, pool, nodes = _get_job(core_job_operations, job_id) return models.Job(job, tasks, pool, nodes) except BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e))