def single_run(policy, trace, size=4, changes={}, name=None, save=True, reuse=False, verbose=False, readonly=False): name = name if name else policy policy = Policy[policy] trace = Trace[trace] if 0 < size < 9: size = trace.typical_caches()[size - 1] conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format( os.sep) conf_file = conf_path + 'application.conf' if not os.path.exists(output_csvs_path): os.makedirs(output_csvs_path) run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava' # run_simulator = './gradlew simulator:run' if os.path.exists(conf_file): conf = ConfigFactory.parse_file(conf_file) else: conf = ConfigFactory.parse_string(""" caffeine { simulator { } } """) simulator = conf['caffeine']['simulator'] simulator.put('files.paths', [resources_path + trace.format() + os.sep + trace.file()]) simulator.put('files.format', trace.value['format']) simulator.put('maximum-size', size) simulator.put('policies', [policy.value]) simulator.put('admission', [Admission.ALWAYS.value]) simulator.put('report.format', 'csv') simulator.put( 'report.output', output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name)) for k, v in changes.items(): simulator.put(k, v) with open(conf_file, 'w') as f: f.write(HOCONConverter.to_hocon(conf)) if (not reuse or not os.path.isfile( simulator['report']['output'])) and not readonly: call(run_simulator, shell=True, cwd=caffeine_root, stdout=subprocess.DEVNULL if not verbose else None) with open(simulator['report']['output'], 'r') as csvfile: reader = csv.DictReader(csvfile) results = {line['Policy']: float(line['Hit rate']) for line in reader} if not save: os.remove(simulator['report']['output']) return results if len(results) != 1 else results[0]
def merge(self, b, update=False): """Merge self with b and then returns a plain string of merged. Args: b: HOCONString, dict, str to be merged. b's `include` statement will always be ignored. update: If True then replace self with a merged one. Returns: String of merged HOCONs. """ if isinstance(b, HOCONString): d = b.to_dict() elif isinstance(b, str): d = HOCONString(b).to_dict() elif isinstance(b, dict): d = b else: raise TypeError('Unsupported type {t}'.format(t=type(b))) self_d = self.to_dict() merge_dict(self_d, d) hocon = ConfigFactory.from_dict(self_d) hocon_str = HOCONConverter.to_hocon(hocon) if update: self._hocon_str = hocon_str return HOCONString(hocon_str).get_contents()
def save(self, output_dir: Union[Path, str]) -> bool: d = Path(output_dir) / instance_full_classname(self) if not os.path.exists(d): os.makedirs(d) logger.info(f"Saving Config to {d / 'config.conf'}") with open(d / "config.conf", "w") as f: f.write(HOCONConverter.to_hocon(self.config)) return True
def oci_image_bundle_conf(args, component_name, oci_manifest, oci_config): conf = ConfigFactory.parse_string('') load_bundle_args_into_conf(conf, args) if 'annotations' in oci_manifest and oci_manifest[ 'annotations'] is not None: annotations_tree = conf.get('annotations') for key in sorted(oci_manifest['annotations']): annotations_tree.put(key, oci_manifest['annotations'][key]) endpoints_tree = ConfigTree() oci_tree = ConfigTree() oci_tree.put('description', args.component_description) oci_tree.put('file-system-type', 'oci-image') oci_tree.put('start-command', ['ociImageTag', args.tag]) oci_tree.put('endpoints', endpoints_tree) components_tree = ConfigTree() components_tree.put(component_name, oci_tree) if args.use_default_endpoints and 'config' in oci_config and 'ExposedPorts' in oci_config[ 'config']: check_arguments = [] for exposed_port in sorted(oci_config['config']['ExposedPorts']): type_parts = exposed_port.split('/', 1) port = int(type_parts[0]) protocol = type_parts[1] if len(type_parts) > 1 else 'tcp' name = '{}-{}-{}'.format(component_name, protocol, port) check_arguments.append('${}_HOST'.format( re.sub('\\W', '_', name.upper()))) entry_tree = ConfigTree() entry_tree.put('bind-protocol', protocol) entry_tree.put('bind-port', port) entry_tree.put('service-name', name) endpoints_tree.put(name, entry_tree) if len(check_arguments) > 0: oci_check_tree = ConfigTree() oci_check_tree.put('description', 'Status check for oci-image component') oci_check_tree.put('file-system-type', 'universal') oci_check_tree.put('start-command', ['check'] + check_arguments) oci_check_tree.put('endpoints', {}) components_tree = ConfigTree() components_tree.put(component_name, oci_tree) components_tree.put('{}-status'.format(component_name), oci_check_tree) conf.put('components', components_tree) return HOCONConverter.to_hocon(conf)
def _load_custom_config(run_config): """Load custom configuration input HOCON file for cromwell. """ from pyhocon import ConfigFactory, HOCONConverter, ConfigTree conf = ConfigFactory.parse_file(run_config) out = {} if "database" in conf: out["database"] = HOCONConverter.to_hocon(ConfigTree({"database": conf.get_config("database")})) return out
def run_one_task(self, queue: Text, task_id: Text, worker_args=None): task_data = self._session.api_client.tasks.get_all(id=[task_id])[0] # push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler try: self._session.api_client.tasks.enqueue( task_id, queue=self.k8s_pending_queue_name, status_reason='k8s pending scheduler') except Exception as e: self.log.error( "ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}" .format(task_id, self.k8s_pending_queue_name, e)) return if task_data.execution.docker_cmd: docker_image = task_data.execution.docker_cmd else: docker_image = str( os.environ.get("TRAINS_DOCKER_IMAGE") or self._session.config.get("agent.default_docker.image", "nvidia/cuda")) # take the first part, this is the docker image name (not arguments) docker_image = docker_image.split()[0] create_trains_conf = "echo '{}' >> ~/trains.conf && ".format( HOCONConverter.to_hocon(self._session.config._config)) if callable(self.kubectl_cmd): kubectl_cmd = self.kubectl_cmd(task_id, docker_image, queue, task_data) else: kubectl_cmd = self.kubectl_cmd.format(task_id=task_id, docker_image=docker_image, queue_id=queue) # make sure we gave a list if isinstance(kubectl_cmd, str): kubectl_cmd = kubectl_cmd.split() kubectl_cmd += [ "--labels=TRAINS=agent", "--command", "--", "/bin/sh", "-c", create_trains_conf + self.container_bash_script.format(task_id) ] process = subprocess.Popen(kubectl_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() self.log.info("K8s scheduling experiment task id={}".format(task_id)) if error: self.log.error("Running kubectl encountered an error: {}".format( error if isinstance(error, str) else error.decode()))
def dumps(instance: object, out: str): conf = __generate(instance, "") if out: if out.lower() == "hocon": return HOCONConverter.to_hocon(conf) if out.lower() == "yaml": return HOCONConverter.to_yaml(conf) if out.lower() == "json": return HOCONConverter.to_json(conf) if out.lower() == "properties": return HOCONConverter.to_properties(conf) return conf
def merge(self, b): if isinstance(b, HOCONString): d = b.to_dict() elif isinstance(b, str): d = HOCONString(b).to_dict() elif isinstance(b, dict): d = b else: raise TypeError('Unsupported type {t}'.format(t=type(b))) self_d = self.to_dict() merge_dict(self_d, d) hocon = ConfigFactory.from_dict(self_d) return self._include + '\n' + HOCONConverter.to_hocon(hocon)
def from_dict(cls, d, include=''): """Create HOCONString from dict. Args: include: `include` statement to be added to the top of the HOCONString. """ hocon = ConfigFactory.from_dict(d) hocon_str = HOCONConverter.to_hocon(hocon) if include: if not is_valid_include(include): raise ValueError( 'Wrong HOCON include format. {include}'.format( include=include)) hocon_str = NEW_LINE.join([include, hocon_str]) return cls(hocon_str=hocon_str)
def from_dict(cls, d, include=''): hocon = ConfigFactory.from_dict(d) hocon_str = HOCONConverter.to_hocon(hocon) if include: hocon_str = include + '\n' + hocon_str return cls(hocon_str=hocon_str)
def saveHoconFile(pathFileNameExt, parameterDict): paramHocon = ConfigFactory.from_dict(parameterDict) paramStr = HOCONConverter.to_hocon(paramHocon,indent=4) with open(pathFileNameExt, "w", encoding='utf-8') as fd: fd.write(paramStr)
def __get_backend_conf_str(self): """ Initializes the following backend stanzas, which are defined in "backend" {} in a Cromwell's backend configuration file: 1) local: local backend 2) gc: Google Cloud backend (optional) 3) aws: AWS backend (optional) 4) slurm: SLURM (optional) 5) sge: SGE (optional) 6) pbs: PBS (optional) Also, initializes the following common non-"backend" stanzas: a) common: base stanzas b) mysql: connect to MySQL (optional) Then converts it to a HOCON string """ # init backend dict backend_dict = {} # common stanza for backend conf file merge_dict( backend_dict, CaperBackendCommon( port=self._port, disable_call_caching=self._disable_call_caching, max_concurrent_workflows=self._max_concurrent_workflows)) # local backend merge_dict( backend_dict, CaperBackendLocal(out_dir=self._out_dir, concurrent_job_limit=self._max_concurrent_tasks)) # GC if self._gcp_prj is not None and self._out_gcs_bucket is not None: merge_dict( backend_dict, CaperBackendGCP( gcp_prj=self._gcp_prj, out_gcs_bucket=self._out_gcs_bucket, concurrent_job_limit=self._max_concurrent_tasks)) # AWS if self._aws_batch_arn is not None and self._aws_region is not None \ and self._out_s3_bucket is not None: merge_dict( backend_dict, CaperBackendAWS( aws_batch_arn=self._aws_batch_arn, aws_region=self._aws_region, out_s3_bucket=self._out_s3_bucket, concurrent_job_limit=self._max_concurrent_tasks)) # SLURM merge_dict( backend_dict, CaperBackendSLURM(out_dir=self._out_dir, partition=self._slurm_partition, account=self._slurm_account, extra_param=self._slurm_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # SGE merge_dict( backend_dict, CaperBackendSGE(out_dir=self._out_dir, pe=self._sge_pe, queue=self._sge_queue, extra_param=self._sge_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # PBS merge_dict( backend_dict, CaperBackendPBS(out_dir=self._out_dir, queue=self._pbs_queue, extra_param=self._pbs_extra_param, concurrent_job_limit=self._max_concurrent_tasks)) # Database if self._no_file_db is not None and self._no_file_db: file_db = None else: file_db = self._file_db merge_dict( backend_dict, CaperBackendDatabase(file_db=file_db, mysql_ip=self._mysql_db_ip, mysql_port=self._mysql_db_port, mysql_user=self._mysql_db_user, mysql_password=self._mysql_db_password, db_timeout=self._db_timeout)) # set header for conf ("include ...") assert (Caper.BACKEND_CONF_HEADER.endswith('\n')) lines_header = [Caper.BACKEND_CONF_HEADER] # override with user-specified backend.conf if exists if self._backend_file is not None: lines_wo_header = [] with open(CaperURI(self._backend_file).get_local_file(), 'r') as fp: for line in fp.readlines(): # find header and exclude if re.findall(Caper.RE_PATTERN_BACKEND_CONF_HEADER, line): if line not in lines_header: lines_header.append(line) else: lines_wo_header.append(line) # parse HOCON to JSON to dict c = ConfigFactory.parse_string(''.join(lines_wo_header)) j = HOCONConverter.to_json(c) d = json.loads(j) # apply to backend conf merge_dict(backend_dict, d) # use default backend (local) if not specified if self._backend is not None: backend_dict['backend']['default'] = self._backend else: backend_dict['backend']['default'] = Caper.DEFAULT_BACKEND # dict to HOCON (excluding header) backend_hocon = ConfigFactory.from_dict(backend_dict) # write header to HOCON string backend_str = ''.join(lines_header) # convert HOCON to string backend_str += HOCONConverter.to_hocon(backend_hocon) return backend_str