def __init__(self, job_name="cloudal", **kwargs): self.keep_alive = kwargs.get('keep_alive') self.out_of_chart = kwargs.get('out_of_chart') self.oar_job_ids = kwargs.get('oar_job_ids') self.config_file_path = kwargs.get('config_file_path') self.configs = kwargs.get('configs') self.is_reservation = kwargs.get('is_reservation') self.no_deploy_os = kwargs.get('no_deploy_os') self.job_name = job_name self.max_deploy = MAX_RETRY_DEPLOY """self.oar_result containts the list of tuples (oar_job_id, site_name) that identifies the reservation on each site, which can be retrieved from the command line arguments or from make_reservation() """ self.oar_result = list() """ TODO: + write function to check all nodes in a job is alive + modify make_reservation function to accept error_hosts -> make replacement reservation or cancel program or ignore """ if self.oar_job_ids is not None: logger.info('Checking the given oar_job_id is valid or not') for each in self.oar_job_ids.split(','): site_name, oar_job_id = each.split(':') oar_job_id = int(oar_job_id) # check validity of oar_job_id job_info = get_oar_job_info(oar_job_id=oar_job_id, frontend=site_name) if job_info is None or len(job_info) == 0: logger.error( "Job id: %s in %s is not a valid Grid5000 oar_job_id" % (oar_job_id, site_name)) logger.error( "Please rerun the script with a correct oar_job_id") exit() self.oar_result.append((int(oar_job_id), str(site_name))) if self.config_file_path and not self.configs: self.configs = parse_config_file(self.config_file_path) return elif self.configs and isinstance(self.configs, dict): logger.debug("Use configs instead of config file") elif self.config_file_path is None: logger.error( "Please provide at least a provisioning config file or oar_job_id." ) exit() else: super(g5k_provisioner, self).__init__(config_file_path=self.config_file_path) # parse clusters into correct format to be used in g5k self.clusters = { each['cluster']: each['n_nodes'] for each in self.configs['clusters'] }
def test_define_parameters(): dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, "../../test_data/test.yaml") result = parse_config_file(file_path) actual = define_parameters(result['parameters']) # assert result is None assert isinstance(actual, dict) assert actual['iteration'] == range(1, 5) assert actual['duration'] == [10] assert actual['workloads'] == ['write']
def run(self): logger.debug('Parse and convert configs for G5K provisioner') self.configs = parse_config_file(self.args.config_file_path) kube_master_site = self.create_configs() logger.info('''Your topology: Antidote DCs: %s n_antidotedb_per_DC: %s ''' % (len(self.configs['exp_env']['antidote_clusters']), self.configs['exp_env']['n_antidotedb_per_dc'])) kube_namespace = 'elmerfs-exp' self.setup_env(kube_master_site, kube_namespace)
def run(self): logger.debug("Parse and convert configs for G5K provisioner") self.configs = parse_config_file(self.args.config_file_path) kube_master_site = self.create_configs() logger.info("""Your topology: %s""" % self.configs["exp_env"]["antidote_clusters"]) # Logarithmic scale interval of latency if self.configs["parameters"][ "latency_interval"] == "logarithmic scale": start, end = self.configs["parameters"]["latency"] latency = [start, end] log_start = int(math.ceil(math.log(start))) log_end = int(math.ceil(math.log(end))) for i in range(log_start, log_end): latency.append(int(math.exp(i))) latency.append(int(math.exp(i + 0.5))) del self.configs["parameters"]["latency_interval"] self.configs["parameters"]["latency"] = list(set(latency)) if self.configs["parameters"]["benchmarks"] == "performance": self.configs["parameters"]["n_nodes_run_per_dc"] = list( range( 1, self.configs["exp_env"]["antidote_clusters"][0] ["n_antidotedb_per_dc"] + 1)) sweeper = create_combs_queue( result_dir=self.configs["exp_env"]["results_dir"], parameters=self.configs["parameters"], ) kube_namespace = "elmerfs-exp" oar_job_ids = None while len(sweeper.get_remaining()) > 0: if oar_job_ids is None: oar_job_ids, kube_master, elmerfs_hosts = self.setup_env( kube_master_site, kube_namespace) comb = sweeper.get_next() sweeper = self.run_workflow( elmerfs_hosts=elmerfs_hosts, kube_master=kube_master, kube_namespace=kube_namespace, comb=comb, sweeper=sweeper, ) if not is_job_alive(oar_job_ids): oardel(oar_job_ids) oar_job_ids = None logger.info("Finish the experiment!!!")
def run(self): logger.debug('Parse and convert configs for G5K provisioner') self.configs = parse_config_file(self.args.config_file_path) # Add the number of Antidote DC as a parameter self.configs['parameters']['n_dc'] = len( self.configs['exp_env']['clusters']) logger.debug('Normalize the parameter space') self.normalized_parameters = define_parameters( self.configs['parameters']) logger.debug('Normalize the given configs') kube_master_site = self.create_configs() logger.info('''Your largest topology: Antidote DCs: %s n_antidotedb_per_DC: %s ''' % (len(self.configs['exp_env']['clusters']), max(self.normalized_parameters['n_nodes_per_dc']))) sweeper = create_combs_queue( result_dir=self.configs['exp_env']['results_dir'], parameters=self.configs['parameters'], ) kube_namespace = 'elmerfs-exp' oar_job_ids = None while len(sweeper.get_remaining()) > 0: if oar_job_ids is None: oar_job_ids, kube_master = self.setup_env( kube_master_site, kube_namespace) comb = sweeper.get_next() sweeper = self.run_workflow( kube_master=kube_master, kube_namespace=kube_namespace, comb=comb, sweeper=sweeper, ) if not is_job_alive(oar_job_ids): oardel(oar_job_ids) oar_job_ids = None logger.info('Finish the experiment!!!')
def test_parse_config_file_valid_input(): expected = { 'walltime': 23400, 'starttime': None, 'cloud_provider_image': 'debian10-x64-big', 'clusters': [{ 'cluster': 'dahu', 'n_nodes': 3 }], 'parameters': { 'iteration': ['1..4'], 'duration': 10, 'workloads': 'write' } } dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(dir_path, "../test_data/test.yaml") result = parse_config_file(file_path) assert result == expected
def run(self): logger.debug('Parse and convert configs for OVH provisioner') self.configs = parse_config_file(self.args.config_file_path) # Add the number of Antidote DC as a parameter self.configs['parameters']['n_dc'] = len( self.configs['exp_env']['clusters']) logger.debug('Normalize the parameter space') self.normalized_parameters = define_parameters( self.configs['parameters']) logger.debug('Normalize the given configs') kube_master_site = self.create_configs() logger.info('''Your largest topology: Antidote DCs: %s n_antidotedb_per_DC: %s n_fmke_client_per_DC: %s ''' % (len(self.configs['exp_env']['clusters']), max(self.normalized_parameters['n_nodes_per_dc']), max(self.normalized_parameters['n_fmke_client_per_dc']))) logger.info('Creating the combination list') sweeper = create_paramsweeper( result_dir=self.configs['exp_env']['results_dir'], parameters=self.normalized_parameters) kube_namespace = 'fmke-exp' node_ids_file = None while len(sweeper.get_remaining()) > 0: if node_ids_file is None: kube_master, node_ids_file = self.setup_env( kube_master_site, kube_namespace) comb = sweeper.get_next() sweeper = self.run_exp_workflow(kube_namespace=kube_namespace, kube_master=kube_master, comb=comb, sweeper=sweeper) # if not is_nodes_alive(node_ids_file): # node_ids_file = None logger.info('Finish the experiment!!!')
def run(self): logger.debug('Parse and convert configs for G5K provisioner') self.configs = parse_config_file(self.args.config_file_path) kube_master_site = self.create_configs() logger.info('''Your topology: Antidote DCs: %s n_antidotedb_per_DC: %s n_fmke_per_DC: %s n_fmke_client_per_DC: %s ''' % ( len(self.configs['exp_env']['clusters']), self.configs['exp_env']['n_antidotedb_per_dc'], self.configs['exp_env']['n_fmke_app_per_dc'], self.configs['exp_env']['n_fmke_client_per_dc']) ) logger.debug('Creating the combination list') sweeper = create_combs_queue(result_dir=self.configs['exp_env']['results_dir'], parameters=self.configs['parameters']) kube_namespace = 'fmke-exp' oar_job_ids = None while len(sweeper.get_remaining()) > 0: if oar_job_ids is None: kube_master, oar_job_ids = self.setup_env(kube_master_site, kube_namespace) comb = sweeper.get_next() sweeper = self.run_workflow(kube_namespace=kube_namespace, kube_master=kube_master, comb=comb, sweeper=sweeper) if not is_job_alive(oar_job_ids): oardel(oar_job_ids) oar_job_ids = None logger.info('Finish the experiment!!!')
def run(self): logger.debug('Parse and convert configs for G5K provisioner') self.configs = parse_config_file(self.args.config_file_path) logger.debug('Creating the combination list') sweeper = create_combs_queue( result_dir=self.configs['exp_env']['results_dir'], parameters=self.configs['parameters']) oar_job_ids = None logger.info('Running the experiment workflow') while len(sweeper.get_remaining()) > 0: if oar_job_ids is None: logger.info('Setting the experiment environment') oar_job_ids = self.setup_env() comb = sweeper.get_next() sweeper = self.run_workflow(comb=comb, sweeper=sweeper) if not is_job_alive(oar_job_ids): oardel(oar_job_ids) oar_job_ids = None logger.info('Finish the experiment!!!')
def test_parse_config_file_wrong_input(file_path, message): with pytest.raises(IOError) as exc_info: parse_config_file(file_path) assert message in str(exc_info)
def __init__(self, config_file_path): self.configs = parse_config_file(config_file_path)