class Executor: """Integrates EasyVVUQ and QCG Pilot Job manager Executor allows to process the most demanding operations of EasyVVUQ in parallel using QCG Pilot Job. """ def __init__(self): self._qcgpjm = None self._tasks = {} self._qcgpj_tempdir = "." def set_manager(self, qcgpjm): """Sets existing QCG Pilot Job Manager as the Executor's engine Parameters ---------- qcgpjm : qcg.appscheduler.api.manager.Manager Existing instance of a QCG Pilot Job Manager Returns ------- None """ self._qcgpjm = qcgpjm print("Available resources:\n%s\n" % str(self._qcgpjm.resources())) def create_manager(self, dir=".", resources=None, reserve_core=False, log_level='debug'): """Creates new QCG Pilot Job Manager and sets is as the Executor's engine Parameters ---------- dir : str The path to the directory where Executor should init QCG Pilot Job Manager. Inside dir a subdirectory for workdir of QCG PJ manager will be created resources : str, optional The resources to use. If specified forces usage of Local mode of QCG Pilot Job Manager. The format is compliant with the NODES format of QCG Pilot Job, i.e.: [node_name:]cores_on_node[,node_name2:cores_on_node][,...]. Eg. to run on 4 cores regardless the node use `resources="4"` to run on 2 cores of node_1 and on 3 cores of node_2 use `resources="node_1:2,node_2:3"` reserve_core : bool, optional If True reserves a core for QCG Pilot Job manager instance, by default QCG Pilot Job Manager shares a core with computing tasks Parameters log_level : str, optional Logging level for QCG Pilot Job Manager (for both service and client part). Returns ------- None """ # ---- QCG PILOT JOB INITIALISATION --- # set QCG-PJ temp directory self._qcgpj_tempdir = mkdtemp(None, ".qcgpj-", dir) log_level = log_level.upper() try: service_log_level = ServiceLogLevel[log_level].value except KeyError: service_log_level = ServiceLogLevel.DEBUG.value try: client_log_level = ClientLogLevel[log_level].value except KeyError: client_log_level = ClientLogLevel.DEBUG.value client_conf = { 'log_file': self._qcgpj_tempdir + '/api.log', 'log_level': client_log_level } common_args = ['--log', service_log_level, '--wd', self._qcgpj_tempdir] args = common_args if resources: args.append('--nodes') args.append(str(resources)) if reserve_core: args.append('--system-core') # create QCGPJ Manager (service part) self._qcgpjm = LocalManager(args, client_conf) def print_resources_info(self): print("Available resources:\n%s\n" % str(self._qcgpjm.resources())) def add_task(self, task): """ Add a task to execute with QCG PJ Parameters ---------- task: easypj.Task The task that will be added to the execution workflow Returns ------- None """ self._tasks[task.get_name()] = task def _get_encoding_task(self, campaign, run): task = self._tasks.get(TaskType.ENCODING) requirements = task.get_requirements().get_resources() key = run[0] enc_args = [ campaign.db_type, campaign.db_location, 'FALSE', campaign.campaign_name, campaign._active_app_name, key ] encode_task = { "name": 'encode_' + key, "execution": { "exec": 'easyvvuq_encode', "args": enc_args, "wd": self._qcgpj_tempdir, "stdout": self._qcgpj_tempdir + '/encode_' + key + '.stdout', "stderr": self._qcgpj_tempdir + '/encode_' + key + '.stderr' } } encode_task.update(requirements) return encode_task def _get_exec_task(self, campaign, run): task = self._tasks.get(TaskType.EXECUTION) application = task.get_params().get("application") requirements = task.get_requirements().get_resources() key = run[0] run_dir = run[1]['run_dir'] exec_args = [run_dir, 'easyvvuq_app', application] execute_task = { "name": 'execute_' + key, "execution": { "exec": 'easyvvuq_execute', "args": exec_args, "wd": self._qcgpj_tempdir, "stdout": self._qcgpj_tempdir + '/execute_' + key + '.stdout', "stderr": self._qcgpj_tempdir + '/execute_' + key + '.stderr' }, "dependencies": { "after": ["encode_" + key] } } execute_task.update(requirements) return execute_task def _get_encoding_and_exec_task(self, campaign, run): task = self._tasks.get(TaskType.ENCODING_AND_EXECUTION) application = task.get_params().get("application") requirements = task.get_requirements().get_resources() key = run[0] run_dir = run[1]['run_dir'] args = [ campaign.db_type, campaign.db_location, 'FALSE', campaign.campaign_name, campaign._active_app_name, key, run_dir, 'easyvvuq_app', application ] encode_execute_task = { "name": 'encode_execute_' + key, "execution": { "exec": 'easyvvuq_encode_execute', "args": args, "wd": self._qcgpj_tempdir, "stdout": self._qcgpj_tempdir + '/encode_execute_' + key + '.stdout', "stderr": self._qcgpj_tempdir + '/encode_execute_' + key + '.stderr' } } encode_execute_task.update(requirements) return encode_execute_task def _get_exec_only_task(self, campaign, run): task = self._tasks.get(TaskType.EXECUTION) application = task.get_params().get("application") requirements = task.get_requirements().get_resources() key = run[0] run_dir = run[1]['run_dir'] exec_args = [run_dir, 'easyvvuq_app', application] execute_task = { "name": 'execute_' + key, "execution": { "exec": 'easyvvuq_execute', "args": exec_args, "wd": self._qcgpj_tempdir, "stdout": self._qcgpj_tempdir + '/execute_' + key + '.stdout', "stderr": self._qcgpj_tempdir + '/execute_' + key + '.stderr' } } execute_task.update(requirements) return execute_task def run(self, campaign, submit_order=SubmitOrder.RUN_ORIENTED): """ Executes demanding parts of EasyVVUQ campaign with QCG Pilot Job A user may choose the preferred execution scheme for the given scenario. campaign : easyvvuq.campaign The campaign object that would be processed. It has to be previously initialised. submit_order: easypj.SubmitOrder EasyVVUQ tasks submission order """ # ---- EXECUTION --- # Execute encode -> execute for each run using QCG-PJ self.__submit_jobs(campaign, submit_order) # wait for completion of all PJ tasks self._qcgpjm.wait4all() print("Syncing state of campaign after execution of PJ") def update_status(run_id, run_data): campaign.campaign_db.set_run_statuses([run_id], uq.constants.Status.ENCODED) campaign.call_for_each_run(update_status, status=uq.constants.Status.NEW) def terminate_manager(self): self._qcgpjm.finish() self._qcgpjm.stopManager() self._qcgpjm.cleanup() def __submit_jobs(self, campaign, submit_order): print("Starting submission of tasks to QCG Pilot Job Manager") if submit_order == SubmitOrder.RUN_ORIENTED_CONDENSED: for run in campaign.list_runs(): self._qcgpjm.submit(Jobs().addStd( self._get_encoding_and_exec_task(campaign, run))) elif submit_order == SubmitOrder.RUN_ORIENTED: for run in campaign.list_runs(): self._qcgpjm.submit(Jobs().addStd( self._get_encoding_task(campaign, run))) self._qcgpjm.submit(Jobs().addStd( self._get_exec_task(campaign, run))) elif submit_order == SubmitOrder.PHASE_ORIENTED: for run in campaign.list_runs(): self._qcgpjm.submit(Jobs().addStd( self._get_encoding_task(campaign, run))) for run in campaign.list_runs(): self._qcgpjm.submit(Jobs().addStd( self._get_exec_task(campaign, run))) elif submit_order == SubmitOrder.EXEC_ONLY: for run in campaign.list_runs(): self._qcgpjm.submit(Jobs().addStd( self._get_exec_only_task(campaign, run)))
"args": exec_args, "wd": cwd, "stdout": my_campaign.campaign_dir + '/execute_' + key + '.stdout', "stderr": my_campaign.campaign_dir + '/execute_' + key + '.stderr' }, "resources": { "numCores": { "exact": 1 } }, "dependencies": { "after": ["encode_" + key] } } m.submit(Jobs().addStd(encode_task)) m.submit(Jobs().addStd(execute_task)) # Wait for completion of all PJ tasks and terminate the PJ manager print(">>> Wait for completion of all PJ tasks") m.wait4all() m.finish() m.stopManager() m.cleanup() print(">>> Syncing state of campaign after execution of PJ") def update_status(run_id, run_data): my_campaign.campaign_db.set_run_statuses([run_id], uq.constants.Status.ENCODED)
def test_cooling_pj(): print("Job directory: " + jobdir) print("Temporary directory: " + tmpdir) # ---- CAMPAIGN INITIALISATION --- print("Initializing Campaign") # Set up a fresh campaign called "cooling" my_campaign = uq.Campaign(name='cooling', work_dir=tmpdir) # Define parameter space params = { "temp_init": { "type": "float", "min": 0.0, "max": 100.0, "default": 95.0 }, "kappa": { "type": "float", "min": 0.0, "max": 0.1, "default": 0.025 }, "t_env": { "type": "float", "min": 0.0, "max": 40.0, "default": 15.0 }, "out_file": { "type": "string", "default": "output.csv" } } output_filename = params["out_file"]["default"] output_columns = ["te", "ti"] # Create an encoder, decoder and collation element for PCE test app encoder = uq.encoders.GenericEncoder(template_fname=jobdir + '/tests/cooling/cooling.template', delimiter='$', target_filename='cooling_in.json') decoder = uq.decoders.SimpleCSV(target_filename=output_filename, output_columns=output_columns, header=0) collater = uq.collate.AggregateSamples(average=False) # Add the PCE app (automatically set as current app) my_campaign.add_app(name="cooling", params=params, encoder=encoder, decoder=decoder, collater=collater) vary = {"kappa": cp.Uniform(0.025, 0.075), "t_env": cp.Uniform(15, 25)} # Create the sampler if uqmethod == 'pce': my_sampler = uq.sampling.PCESampler(vary=vary, polynomial_order=2) else: my_sampler = uq.sampling.QMCSampler(vary=vary, n_samples=10) # Associate the sampler with the campaign my_campaign.set_sampler(my_sampler) # Will draw all (of the finite set of samples) my_campaign.draw_samples() # ---- QCG PILOT JOB INITIALISATION --- # set QCG-PJ temp directory qcgpj_tempdir = mkdtemp(None, ".qcgpj-", my_campaign.campaign_dir) # establish available resources cores = 4 # switch on debugging of QCGPJ API (client part) client_conf = { 'log_file': qcgpj_tempdir + '/api.log', 'log_level': 'DEBUG' } # create local LocalManager (service part) m = LocalManager( ['--log', 'debug', '--nodes', str(cores), '--wd', qcgpj_tempdir], client_conf) # This can be used for execution of the test using a separate (non-local) instance of PJManager # # get available resources # res = m.resources() # remove all jobs if they are already in PJM # (required when executed using the same QCG-Pilot Job Manager) # m.remove(m.list().keys()) print("Available resources:\n%s\n" % str(m.resources())) # ---- EXECUTION --- # Execute encode -> execute for each run using QCG-PJ print("Starting submission of tasks to QCG Pilot Job Manager") for run in my_campaign.list_runs(): key = run[0] run_dir = run[1]['run_dir'] enc_args = [ my_campaign.db_type, my_campaign.db_location, 'FALSE', "cooling", "cooling", key ] exec_args = [ run_dir, 'easyvvuq_app', 'python3 ' + jobdir + "/tests/cooling/cooling_model.py", "cooling_in.json" ] encode_task = { "name": 'encode_' + key, "execution": { "exec": 'easyvvuq_encode', "args": enc_args, "wd": my_campaign.campaign_dir, "stdout": my_campaign.campaign_dir + '/encode_' + key + '.stdout', "stderr": my_campaign.campaign_dir + '/encode_' + key + '.stderr' }, "resources": { "numCores": { "exact": 1 } } } execute_task = { "name": 'execute_' + key, "execution": { "exec": 'easyvvuq_execute', "args": exec_args, "wd": my_campaign.campaign_dir, "stdout": my_campaign.campaign_dir + '/execute_' + key + '.stdout', "stderr": my_campaign.campaign_dir + '/execute_' + key + '.stderr' }, "resources": { "numCores": { "exact": 1 } }, "dependencies": { "after": ["encode_" + key] } } m.submit(Jobs().addStd(encode_task)) m.submit(Jobs().addStd(execute_task)) # wait for completion of all PJ tasks and terminate the PJ manager m.wait4all() m.finish() m.stopManager() m.cleanup() print("Syncing state of campaign after execution of PJ") def update_status(run_id, run_data): my_campaign.campaign_db.set_run_statuses([run_id], uq.constants.Status.ENCODED) my_campaign.call_for_each_run(update_status, status=uq.constants.Status.NEW) print("Collating results") my_campaign.collate() # Post-processing analysis print("Making analysis") if uqmethod == 'pce': analysis = uq.analysis.PCEAnalysis(sampler=my_sampler, qoi_cols=output_columns) else: analysis = uq.analysis.QMCAnalysis(sampler=my_sampler, qoi_cols=output_columns) my_campaign.apply_analysis(analysis) results = my_campaign.get_last_analysis() # Get Descriptive Statistics stats = results['statistical_moments']['te'] print("Processing completed") return stats
from qcg.appscheduler.api.manager import Manager from qcg.appscheduler.api.manager import LocalManager from qcg.appscheduler.api.job import Jobs m = LocalManager(cfg={'log_level': 'DEBUG'}, server_args=['--log', 'debug']) # m = Manager(cfg={'log_level': 'DEBUG'}, server_args=['--log', 'debug']) # get available resources print("available resources:\n%s\n" % str(m.resources())) # submit jobs and save their names in 'ids' list jobs = Jobs() $submitted_jobs_list ids = m.submit(jobs) # wait until submited jobs finish m.wait4(ids) # get detailed information about submited and finished jobs print("jobs details:\n%s\n" % str(m.info(ids))) m.finish() m.stopManager() m.cleanup()
def test_pce_pj(tmpdir): print("Running in directory: " + cwd) # establish available resources cores = 1 # set location of log file # client_conf = {'log_file': tmpdir.join('api.log'), 'log_level': 'DEBUG'} # switch on debugging (by default in api.log file) client_conf = {'log_level': 'DEBUG'} # switch on debugging (by default in api.log file) m = LocalManager(['--nodes', str(cores)], client_conf) # This can be used for execution of the test using a separate (non-local) instance of PJManager # # get available resources # res = m.resources() # remove all jobs if they are already in PJM # (required when executed using the same QCG-Pilot Job Manager) # m.remove(m.list().keys()) print("Available resources:\n%s\n" % str(m.resources())) print("Initializing Camapign") # Set up a fresh campaign called "pce" my_campaign = uq.Campaign(name='pce', work_dir=tmpdir) # Define parameter space params = { "kappa": { "type": "real", "min": "0.0", "max": "0.1", "default": "0.025" }, "t_env": { "type": "real", "min": "0.0", "max": "40.0", "default": "15.0" }, "out_file": { "type": "str", "default": "output.csv" } } output_filename = params["out_file"]["default"] output_columns = ["te", "ti"] # Create an encoder, decoder and collation element for PCE test app encoder = uq.encoders.GenericEncoder(template_fname=pce_app_dir + '/pce.template', delimiter='$', target_filename='pce_in.json') decoder = uq.decoders.SimpleCSV(target_filename=output_filename, output_columns=output_columns, header=0) # Add the PCE app (automatically set as current app) my_campaign.add_app(name="pce", params=params, encoder=encoder, decoder=decoder) # Create a collation element for this campaign collater = uq.collate.AggregateSamples(average=False) my_campaign.set_collater(collater) # Create the sampler vary = {"kappa": cp.Uniform(0.025, 0.075), "t_env": cp.Uniform(15, 25)} my_sampler = uq.sampling.PCESampler(vary=vary, polynomial_order=1) # Associate the sampler with the campaign my_campaign.set_sampler(my_sampler) # Will draw all (of the finite set of samples) my_campaign.draw_samples() # Create & save PJ configurator print("Creating configuration for QCG Pilot Job Manager") PJConfigurator(my_campaign).save() # Execute encode -> execute for each run using QCG-PJ print("Starting submission of tasks to QCG Pilot Job Manager") for key in my_campaign.list_runs(): encode_job = { "name": 'encode_' + key, "execution": { "exec": 'easyvvuq_encode', "args": [my_campaign.campaign_dir, key], "wd": cwd, "env": { "EASYPJ_CONF": easypj_conf }, }, "resources": { "numCores": { "exact": 1 } } } execute_job = { "name": 'execute_' + key, "execution": { "exec": 'easyvvuq_execute', "args": [ my_campaign.campaign_dir, key, 'easyvvuq_app', pce_app_dir + "/pce_model.py", "pce_in.json" ], "wd": cwd, "env": { "EASYPJ_CONF": easypj_conf }, }, "resources": { "numCores": { "exact": 1 } }, "dependencies": { "after": ["encode_" + key] } } m.submit(Jobs().addStd(encode_job)) m.submit(Jobs().addStd(execute_job)) print("Waiting for completion of all QCG PJ tasks") # wait for completion of all PJ tasks and terminate the PJ manager m.wait4all() m.finish() m.stopManager() m.cleanup() print("Collating results") my_campaign.collate() # Update after here # Post-processing analysis print("Making analysis") pce_analysis = uq.analysis.PCEAnalysis(sampler=my_sampler, qoi_cols=output_columns) my_campaign.apply_analysis(pce_analysis) results = my_campaign.get_last_analysis() # Get Descriptive Statistics stats = results['statistical_moments']['te'] sobols = results['sobols_first']['te'] print("Stats: ") print(stats) print("Sobols: ") print(sobols) print("Processing completed")