def single_run(policy,
trace,
size=4,
changes={},
name=None,
save=True,
reuse=False,
verbose=False,
readonly=False):
name = name if name else policy
policy = Policy[policy]
trace = Trace[trace]
if 0 < size < 9:
size = trace.typical_caches()[size - 1]
conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format(
os.sep)
conf_file = conf_path + 'application.conf'
if not os.path.exists(output_csvs_path):
os.makedirs(output_csvs_path)
run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava'
# run_simulator = './gradlew simulator:run'
if os.path.exists(conf_file):
conf = ConfigFactory.parse_file(conf_file)
else:
conf = ConfigFactory.parse_string("""
caffeine {
simulator {
}
}
""")
simulator = conf['caffeine']['simulator']
simulator.put('files.paths',
[resources_path + trace.format() + os.sep + trace.file()])
simulator.put('files.format', trace.value['format'])
simulator.put('maximum-size', size)
simulator.put('policies', [policy.value])
simulator.put('admission', [Admission.ALWAYS.value])
simulator.put('report.format', 'csv')
simulator.put(
'report.output',
output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name))
for k, v in changes.items():
simulator.put(k, v)
with open(conf_file, 'w') as f:
f.write(HOCONConverter.to_hocon(conf))
if (not reuse or not os.path.isfile(
simulator['report']['output'])) and not readonly:
call(run_simulator,
shell=True,
cwd=caffeine_root,
stdout=subprocess.DEVNULL if not verbose else None)
with open(simulator['report']['output'], 'r') as csvfile:
reader = csv.DictReader(csvfile)
results = {line['Policy']: float(line['Hit rate']) for line in reader}
if not save:
os.remove(simulator['report']['output'])
return results if len(results) != 1 else results[0]
def merge(self, b, update=False):
"""Merge self with b and then returns a plain string of merged.
Args:
b:
HOCONString, dict, str to be merged.
b's `include` statement will always be ignored.
update:
If True then replace self with a merged one.
Returns:
String of merged HOCONs.
"""
if isinstance(b, HOCONString):
d = b.to_dict()
elif isinstance(b, str):
d = HOCONString(b).to_dict()
elif isinstance(b, dict):
d = b
else:
raise TypeError('Unsupported type {t}'.format(t=type(b)))
self_d = self.to_dict()
merge_dict(self_d, d)
hocon = ConfigFactory.from_dict(self_d)
hocon_str = HOCONConverter.to_hocon(hocon)
if update:
self._hocon_str = hocon_str
return HOCONString(hocon_str).get_contents()
def save(self, output_dir: Union[Path, str]) -> bool:
d = Path(output_dir) / instance_full_classname(self)
if not os.path.exists(d):
os.makedirs(d)
logger.info(f"Saving Config to {d / 'config.conf'}")
with open(d / "config.conf", "w") as f:
f.write(HOCONConverter.to_hocon(self.config))
return True
def oci_image_bundle_conf(args, component_name, oci_manifest, oci_config):
conf = ConfigFactory.parse_string('')
load_bundle_args_into_conf(conf, args)
if 'annotations' in oci_manifest and oci_manifest[
'annotations'] is not None:
annotations_tree = conf.get('annotations')
for key in sorted(oci_manifest['annotations']):
annotations_tree.put(key, oci_manifest['annotations'][key])
endpoints_tree = ConfigTree()
oci_tree = ConfigTree()
oci_tree.put('description', args.component_description)
oci_tree.put('file-system-type', 'oci-image')
oci_tree.put('start-command', ['ociImageTag', args.tag])
oci_tree.put('endpoints', endpoints_tree)
components_tree = ConfigTree()
components_tree.put(component_name, oci_tree)
if args.use_default_endpoints and 'config' in oci_config and 'ExposedPorts' in oci_config[
'config']:
check_arguments = []
for exposed_port in sorted(oci_config['config']['ExposedPorts']):
type_parts = exposed_port.split('/', 1)
port = int(type_parts[0])
protocol = type_parts[1] if len(type_parts) > 1 else 'tcp'
name = '{}-{}-{}'.format(component_name, protocol, port)
check_arguments.append('${}_HOST'.format(
re.sub('\\W', '_', name.upper())))
entry_tree = ConfigTree()
entry_tree.put('bind-protocol', protocol)
entry_tree.put('bind-port', port)
entry_tree.put('service-name', name)
endpoints_tree.put(name, entry_tree)
if len(check_arguments) > 0:
oci_check_tree = ConfigTree()
oci_check_tree.put('description',
'Status check for oci-image component')
oci_check_tree.put('file-system-type', 'universal')
oci_check_tree.put('start-command', ['check'] + check_arguments)
oci_check_tree.put('endpoints', {})
components_tree = ConfigTree()
components_tree.put(component_name, oci_tree)
components_tree.put('{}-status'.format(component_name),
oci_check_tree)
conf.put('components', components_tree)
return HOCONConverter.to_hocon(conf)
def _load_custom_config(run_config):
"""Load custom configuration input HOCON file for cromwell.
"""
from pyhocon import ConfigFactory, HOCONConverter, ConfigTree
conf = ConfigFactory.parse_file(run_config)
out = {}
if "database" in conf:
out["database"] = HOCONConverter.to_hocon(ConfigTree({"database": conf.get_config("database")}))
return out
def run_one_task(self, queue: Text, task_id: Text, worker_args=None):
task_data = self._session.api_client.tasks.get_all(id=[task_id])[0]
# push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler
try:
self._session.api_client.tasks.enqueue(
task_id,
queue=self.k8s_pending_queue_name,
status_reason='k8s pending scheduler')
except Exception as e:
self.log.error(
"ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}"
.format(task_id, self.k8s_pending_queue_name, e))
return
if task_data.execution.docker_cmd:
docker_image = task_data.execution.docker_cmd
else:
docker_image = str(
os.environ.get("TRAINS_DOCKER_IMAGE")
or self._session.config.get("agent.default_docker.image",
"nvidia/cuda"))
# take the first part, this is the docker image name (not arguments)
docker_image = docker_image.split()[0]
create_trains_conf = "echo '{}' >> ~/trains.conf && ".format(
HOCONConverter.to_hocon(self._session.config._config))
if callable(self.kubectl_cmd):
kubectl_cmd = self.kubectl_cmd(task_id, docker_image, queue,
task_data)
else:
kubectl_cmd = self.kubectl_cmd.format(task_id=task_id,
docker_image=docker_image,
queue_id=queue)
# make sure we gave a list
if isinstance(kubectl_cmd, str):
kubectl_cmd = kubectl_cmd.split()
kubectl_cmd += [
"--labels=TRAINS=agent", "--command", "--", "/bin/sh", "-c",
create_trains_conf + self.container_bash_script.format(task_id)
]
process = subprocess.Popen(kubectl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = process.communicate()
self.log.info("K8s scheduling experiment task id={}".format(task_id))
if error:
self.log.error("Running kubectl encountered an error: {}".format(
error if isinstance(error, str) else error.decode()))
def dumps(instance: object, out: str):
conf = __generate(instance, "")
if out:
if out.lower() == "hocon":
return HOCONConverter.to_hocon(conf)
if out.lower() == "yaml":
return HOCONConverter.to_yaml(conf)
if out.lower() == "json":
return HOCONConverter.to_json(conf)
if out.lower() == "properties":
return HOCONConverter.to_properties(conf)
return conf
def merge(self, b):
if isinstance(b, HOCONString):
d = b.to_dict()
elif isinstance(b, str):
d = HOCONString(b).to_dict()
elif isinstance(b, dict):
d = b
else:
raise TypeError('Unsupported type {t}'.format(t=type(b)))
self_d = self.to_dict()
merge_dict(self_d, d)
hocon = ConfigFactory.from_dict(self_d)
return self._include + '\n' + HOCONConverter.to_hocon(hocon)
def from_dict(cls, d, include=''):
"""Create HOCONString from dict.
Args:
include:
`include` statement to be added to the top of the HOCONString.
"""
hocon = ConfigFactory.from_dict(d)
hocon_str = HOCONConverter.to_hocon(hocon)
if include:
if not is_valid_include(include):
raise ValueError(
'Wrong HOCON include format. {include}'.format(
include=include))
hocon_str = NEW_LINE.join([include, hocon_str])
return cls(hocon_str=hocon_str)
def from_dict(cls, d, include=''):
hocon = ConfigFactory.from_dict(d)
hocon_str = HOCONConverter.to_hocon(hocon)
if include:
hocon_str = include + '\n' + hocon_str
return cls(hocon_str=hocon_str)
def saveHoconFile(pathFileNameExt, parameterDict):
paramHocon = ConfigFactory.from_dict(parameterDict)
paramStr = HOCONConverter.to_hocon(paramHocon,indent=4)
with open(pathFileNameExt, "w", encoding='utf-8') as fd:
fd.write(paramStr)
def __get_backend_conf_str(self):
"""
Initializes the following backend stanzas,
which are defined in "backend" {} in a Cromwell's backend
configuration file:
1) local: local backend
2) gc: Google Cloud backend (optional)
3) aws: AWS backend (optional)
4) slurm: SLURM (optional)
5) sge: SGE (optional)
6) pbs: PBS (optional)
Also, initializes the following common non-"backend" stanzas:
a) common: base stanzas
b) mysql: connect to MySQL (optional)
Then converts it to a HOCON string
"""
# init backend dict
backend_dict = {}
# common stanza for backend conf file
merge_dict(
backend_dict,
CaperBackendCommon(
port=self._port,
disable_call_caching=self._disable_call_caching,
max_concurrent_workflows=self._max_concurrent_workflows))
# local backend
merge_dict(
backend_dict,
CaperBackendLocal(out_dir=self._out_dir,
concurrent_job_limit=self._max_concurrent_tasks))
# GC
if self._gcp_prj is not None and self._out_gcs_bucket is not None:
merge_dict(
backend_dict,
CaperBackendGCP(
gcp_prj=self._gcp_prj,
out_gcs_bucket=self._out_gcs_bucket,
concurrent_job_limit=self._max_concurrent_tasks))
# AWS
if self._aws_batch_arn is not None and self._aws_region is not None \
and self._out_s3_bucket is not None:
merge_dict(
backend_dict,
CaperBackendAWS(
aws_batch_arn=self._aws_batch_arn,
aws_region=self._aws_region,
out_s3_bucket=self._out_s3_bucket,
concurrent_job_limit=self._max_concurrent_tasks))
# SLURM
merge_dict(
backend_dict,
CaperBackendSLURM(out_dir=self._out_dir,
partition=self._slurm_partition,
account=self._slurm_account,
extra_param=self._slurm_extra_param,
concurrent_job_limit=self._max_concurrent_tasks))
# SGE
merge_dict(
backend_dict,
CaperBackendSGE(out_dir=self._out_dir,
pe=self._sge_pe,
queue=self._sge_queue,
extra_param=self._sge_extra_param,
concurrent_job_limit=self._max_concurrent_tasks))
# PBS
merge_dict(
backend_dict,
CaperBackendPBS(out_dir=self._out_dir,
queue=self._pbs_queue,
extra_param=self._pbs_extra_param,
concurrent_job_limit=self._max_concurrent_tasks))
# Database
if self._no_file_db is not None and self._no_file_db:
file_db = None
else:
file_db = self._file_db
merge_dict(
backend_dict,
CaperBackendDatabase(file_db=file_db,
mysql_ip=self._mysql_db_ip,
mysql_port=self._mysql_db_port,
mysql_user=self._mysql_db_user,
mysql_password=self._mysql_db_password,
db_timeout=self._db_timeout))
# set header for conf ("include ...")
assert (Caper.BACKEND_CONF_HEADER.endswith('\n'))
lines_header = [Caper.BACKEND_CONF_HEADER]
# override with user-specified backend.conf if exists
if self._backend_file is not None:
lines_wo_header = []
with open(CaperURI(self._backend_file).get_local_file(),
'r') as fp:
for line in fp.readlines():
# find header and exclude
if re.findall(Caper.RE_PATTERN_BACKEND_CONF_HEADER, line):
if line not in lines_header:
lines_header.append(line)
else:
lines_wo_header.append(line)
# parse HOCON to JSON to dict
c = ConfigFactory.parse_string(''.join(lines_wo_header))
j = HOCONConverter.to_json(c)
d = json.loads(j)
# apply to backend conf
merge_dict(backend_dict, d)
# use default backend (local) if not specified
if self._backend is not None:
backend_dict['backend']['default'] = self._backend
else:
backend_dict['backend']['default'] = Caper.DEFAULT_BACKEND
# dict to HOCON (excluding header)
backend_hocon = ConfigFactory.from_dict(backend_dict)
# write header to HOCON string
backend_str = ''.join(lines_header)
# convert HOCON to string
backend_str += HOCONConverter.to_hocon(backend_hocon)
return backend_str
