def load_settings():
yaml = YAML()
if os.path.exists(DEFAULT_SETTINGS_FILE):
with open(DEFAULT_SETTINGS_FILE, 'r') as f:
settings = dict(yaml.load(f))
if settings['whitelist-enabled'] and settings['blacklist-enabled']:
log('whitelist and blacklist cannot be enabled simultaneously')
return None
# Paths starting and ending with '/'
for i, w in enumerate(settings['whitelist-files']):
if w[0] != '/':
settings['whitelist-files'][i] = '/' + w
if w[-1] != '/':
settings['whitelist-files'][
i] = settings['whitelist-files'][i] + '/'
for i, w in enumerate(settings['blacklist-files']):
if w[0] != '/':
settings['blacklist-files'][i] = '/' + w
if w[-1] != '/':
settings['blacklist-files'][i] = settings[i] + '/'
return settings
with open(DEFAULT_SETTINGS_FILE, 'w+') as f:
yaml.dump(DEFAULT_SETTINGS, f)
return DEFAULT_SETTINGS
def _save(self):
'save configuration in file cache to be restored'
resource_name = self._get_one_resource_value()
namespace = self.client.get_namespace(
self._get_kube_api_resource_type(), resource_name)
kube_instance = self._get_one_resource_value("kube")
if not kube_instance:
kube_instance = 'local'
cache_folder = os.path.join(settings.OMC_KUBE_CACHE_DIR, kube_instance,
namespace, self._get_kube_resource_type())
result = self._read_namespaced_resource(resource_name,
namespace,
_preload_content=False)
stream = StringIO()
the_result = json.loads(result.data.decode('UTF-8'))
ObjectUtils.delete_node(the_result, 'metadata.creationTimestamp')
ObjectUtils.delete_node(the_result, 'metadata.resourceVersion')
yaml = YAML()
yaml.dump(the_result, stream)
content = stream.getvalue()
make_directory(cache_folder)
with open(os.path.join(cache_folder, resource_name + '.yaml'),
'w') as f:
f.write(content)
def generate_hpsearch():
# TODO documentation
yaml = YAML()
parent_file_path = Path(args.input_file)
parent_yaml = yaml.load(parent_file_path)
hpsearch_config = parent_yaml['hpsearch']
if hpsearch_config.get('is_child', False):
raise RuntimeError(
'This YAML is itself a child config generated for an hyperparameter search.'
)
assert hpsearch_config['type'] == 'random_uniform'
description = hpsearch_config[
'desc'] if 'desc' in hpsearch_config else 'hpsearch'
for trial_idx in range(hpsearch_config['n_trials']):
child_yaml = make_child_config(parent_yaml)
for p in hpsearch_config['params']:
value = generate_random_value(p)
set_item_at_path(child_yaml, p['param'], value)
child_file_path = parent_file_path.with_name(
parent_file_path.stem +
'_{}_{}.yaml'.format(description, trial_idx))
yaml.dump(child_yaml, child_file_path)
def __init__(self,
dir_: str,
file_path_yml: Optional[str] = None,
loglevel=logging.DEBUG):
self.logger = get_logger(self.__class__.__name__, loglevel=loglevel)
# create a yaml file reader
self.yaml = YAML()
# self.yaml.indent(mapping=2, sequence=4, offset=2)
self.yaml.indent(offset=2)
self.yaml.default_flow_style = False
self.yaml.width = 4096
# attributes that change upon changing the starting dir
self.configs = {}
self.reader_data = {}
self.interesting_proteins, self.go_analysis_gene_names = None, None
# properties
self._start_dir = None
self._file_path_yaml = None
# set the specified dirs
self.start_dir = dir_
if file_path_yml is not None:
self.file_path_yaml = file_path_yml
def _resource_completion(self, short_mode=True):
results = []
headers = ('NAME', 'SERVER', 'CLUSTER', 'CONTEXT')
yaml = YAML()
results.append(headers)
if os.path.exists(settings.OMC_KUBE_CONFIG_DIR):
resources = os.listdir(settings.OMC_KUBE_CONFIG_DIR)
for one_resource in resources:
try:
with open(
os.path.join(settings.OMC_KUBE_CONFIG_DIR,
one_resource, 'config')) as f:
content = yaml.load(f)
one_item = (one_resource,
ObjectUtils.get_node(
content, "clusters[0].cluster.server"),
ObjectUtils.get_node(
content, "clusters[0].name"),
ObjectUtils.get_node(
content, "current-context"))
results.append(one_item)
except Exception as e:
logger.error(e, exc_info=True)
return CompletionContent(Formatter.format_completions(results))
def main(args=None):
"""Build a GSC config for running Ploidetect.
Uses GSC standars patient_id and biopsy or libraries.
Ploidetect versions must just be manually specified.
"""
logger.setLevel(logging.INFO)
args = parse_args() if not args else args
if args.output_file and exists(args.output_file):
raise ValueError(f"Output config already exists: '{args.output_file}'")
config = build_config(**vars(args))
if not args.output_file:
YAML().dump(config, sys.stdout)
else:
args.output_file = args.output_file.replace("DERIVED_OUTPUT_DIR",
config["output_dir"])
if exists(args.output_file):
raise ValueError(
f"Output config already exists: '{args.output_file}'")
elif (dirname(
args.output_file)) and not exists(dirname(args.output_file)):
logger.warning(
f"Creating output folder: {dirname(args.output_file)}")
os.makedirs(dirname(args.output_file))
print(f"Writing config to: {abspath(realpath(args.output_file))}")
YAML().dump(config, open(args.output_file, "w"))
def parse_hp(hpfile):
yaml = YAML(typ="safe", pure=True)
with open(hpfile) as hp_raw:
hps = yaml.load(hp_raw)
tf_hps = tf.contrib.training.HParams()
for k, v in hps["default"].items():
tf_hps.add_hparam(k, v)
return tf_hps
def modify(self, mods, inputs=None, outputs=None):
"""
Allows to make modifications to the model. It will create a modified configuration file, build the corresponding model and
set the weights of the current model when possible.
mods: a list of dictionaries. Each dictionary can have as key: delete (in this case the value, which is a path,
is deleted from config), or a config path (in this case, the path value is replaced by the value of the dictionary).
inputs: a list with the names of the inputs
outputs: a list with the names of the outputs
"""
model_weights = self.get_weights()
yaml_loader = YAML()
m_conf = Config(self.core_model.processed_config)
original_keys = list(m_conf.keys())
deep_conf = Config(shallow_to_deep(m_conf))
for mod in mods:
mod_key = list(mod.keys())[0]
mod_value = mod[mod_key]
if mod_key == 'delete':
deep_conf.pop(mod_value)
if mod_value in original_keys:
original_keys.remove(mod_value)
elif '*' in mod_key:
mod_key = mod_key.lstrip('/')
found_paths = [
k for k in deep_conf.to_shallow().keys()
if fnmatch.fnmatch(k, mod_key)
]
for k in found_paths:
k = k.replace('.', '/')
if isinstance(mod_value, str):
deep_conf[k] = yaml_loader.load(mod_value)
else:
deep_conf[k] = mod_value
else:
mod_key = mod_key.replace('.', '/')
if mod_key.split('/')[0] not in deep_conf.keys(
): #This means we are adding a new layer
layer_name = mod_key.split('/')[0]
original_keys.append(layer_name)
deep_conf['{}/name'.format(layer_name)] = layer_name
if isinstance(mod_value, str):
deep_conf[mod_key] = yaml_loader.load(mod_value)
else:
deep_conf[mod_key] = mod_value
new_model_architecture = shallow_to_original_keys(
deep_conf.to_shallow(), original_keys)
model = self.build(processed_config=new_model_architecture,
input_names=inputs,
output_names=outputs)
layer_names = [l.name for l in model.layers]
for k, v in model_weights.items():
if k in layer_names:
layer = model.get_layer(k)
layer.set_weights(v)
self.core_model.model = model
def yaml(self):
'get configuration in yaml format'
resource = self._get_one_resource_value()
namespace = self.client.get_namespace(
self._get_kube_api_resource_type(), resource)
result = self._read_namespaced_resource(resource, namespace)
stream = StringIO()
yaml = YAML()
yaml.dump(result, stream)
console.log(stream.getvalue())
def main():
count = 0
yaml = YAML()
for post in glob.glob(os.path.join(POSTS_PATH, "*.md")):
git_lastmod = subprocess.check_output([
"git", "log", "-1", "--pretty=%ad", "--date=short", post]).strip()
if not git_lastmod:
continue
frontmatter, line_num = get_yaml(post)
meta = yaml.load(frontmatter)
if 'sitemap' in meta:
if 'lastmod' in meta['sitemap']:
if meta['sitemap']['lastmod'] == git_lastmod:
continue
meta['sitemap']['lastmod'] = git_lastmod
else:
meta['sitemap'].insert(0, 'lastmod', git_lastmod)
else:
meta.insert(line_num, 'sitemap', {'lastmod': git_lastmod})
output = 'new.md'
if os.path.isfile(output):
os.remove(output)
with open(output, 'w') as new, open(post, 'r') as old:
new.write("---\n")
yaml.dump(meta, new)
new.write("---\n")
line_num += 2
lines = old.readlines()
for line in lines:
if line_num > 0:
line_num -= 1
continue
else:
new.write(line)
shutil.move(output, post)
count += 1
print("[INFO] update 'lastmod' for: '{}'".format(post))
print("[NOTICE] Success! Update all posts's lastmod.\n")
if count > 0:
subprocess.call(["git", "add", POSTS_PATH])
subprocess.call(["git", "commit", "-m",
"[Automation] Update lastmod of post(s)."])
def read_parameters(param_file):
"""Read and return parameters in .yaml file
Args:
param_file: Full file path of the parameters file
Returns:
YAML (Ruamel) CommentedMap dict-like object
"""
yaml = YAML()
with open(param_file) as yamlfile:
params = yaml.load(yamlfile)
return params
def from_yaml(cls, yaml_str: str) -> 'RunDescriber':
"""
Parse a yaml string (the return of `to_yaml`) into a RunDescriber
object
"""
YAML = cls._ruamel_importer()
yaml = YAML()
# yaml.load returns an OrderedDict, but we need a dict
ser = dict(yaml.load(yaml_str))
return cls.deserialize(ser)
def __init__(self, threadID):
Thread.__init__(self)
self.threadID = threadID
solver = POMCPV
yaml = YAML()
with open('exp_param.yml', 'r') as param:
args = yaml.load(param)
args['ucb_coefficient'] = threadID
np.random.seed(args['seed'])
random.seed(a=args['seed'])
env = RobotModel(args)
self.agent = AgentSMC(env, solver)
def to_yaml(self) -> str:
"""
Output the run description as a yaml string
"""
YAML = self._ruamel_importer()
yaml = YAML()
with io.StringIO() as stream:
yaml.dump(self.serialize(), stream=stream)
output = stream.getvalue()
return output
def dump(self, data, stream=None, **kwargs):
"""
Output data to a given stream.
If no stream is given, then the data is returned as a string.
"""
inefficient = False
if stream is None:
inefficient = True
stream = StringIO()
YAML.dump(self, data, stream, **kwargs)
if inefficient:
return stream.getvalue()
def __init__(self, yaml_file, geojson_file):
with open(yaml_file, 'r') as stream:
yaml = YAML()
config = yaml.load(stream)
# error handling happens in image reader
self.image_list = []
for tile_args in config['tile-list']:
tile_config = config['tile-list'][tile_args]
tile_settings = ImageReader(tile_config, geojson_file)
self.image_list.append(tile_settings)
# parse mosaic settings
if config['mosaic-settings']['build-mosaic'] == True:
try:
self.mosaic_keywords = [
"build-mosaic", "resampling-method", "clip"
]
self.mosaic_settings = self.parse_settings(
self.mosaic_keywords, config['mosaic-settings'])
self.mosaic_settings['image-list'] = []
for i in config['mosaic-settings']['image-list']:
self.mosaic_settings['image-list'].append(
config['mosaic-settings']['image-list'][i])
except Exception:
raise IOError('in YAML file mosaic-settings not defined')
else:
self.mosaic_settings = {}
self.mosaic_settings['build-mosaic'] = False
# parse average settings
if config['average-settings']['compute-average'] is True:
try:
self.average_keywords = [
"compute-average", "include-mosaic", "clip"
]
self.average_settings = self.parse_settings(
self.average_keywords, config['average-settings'])
self.average_settings['image-list'] = []
for i in config['average-settings']['image-list']:
self.average_settings['image-list'].append(
config['average-settings']['image-list'][i])
except Exception:
raise IOError('in YAML file average-settings not defined')
else:
self.average_settings = {}
self.average_settings['compute-average'] = False
def copy_cookiecutter_resume(template_name='easydata'):
"""Make a copy of the cookiecutter replay file in the generated project.
By default, cookiecutter creates a replay directory in a user's
~/.cookiecutter directory. This is largely useless. Easydata dumps
this data to the generated project (also as json) using a jsonify
call, but this doesn't yet help us regenerate the project
automatically. This hook creates a YAML version of those values
in the generated project. This can be used to regenerate the
project by doing a:
>>> cookiecutter --config_file path/to/.easydata.yaml easydata
"""
# relative to root of generated project
src_path = f'.{template_name}.json'
yml_path = f'.{template_name}.yml'
logger.debug(f"Reading cookiecutter replay data from {src_path}")
with open(src_path) as f:
cookiecutter_opts = json.load(f)
yaml_opts = {k:v
for k,v in sorted(cookiecutter_opts.items())
if not k.startswith('_')}
yaml = YAML()
yaml.default_flow_style=False
yaml.width=4096
yaml.indent(offset=4)
logger.debug(f"Dumping cookiecutter replay (YAML) info to {yml_path}")
with open(yml_path, 'w') as fw:
yaml.dump({'default_context': yaml_opts}, fw)
def copy_cookiecutter_resume(template_name='cookiecutter-easydata'):
"""Make a copy of the cookiecutter replay file in the generated project.
By default, cookiecutter creates a replay directory in a user's ~/.cookiecutter
directory. This hook creates a YAML version of those values in the generated project.
This can be used to regenerate the project by doing a:
>>> cookiecutter --config_file path/to/cookiecutter-easydata.yaml cookiecutter-easydata
"""
config_obj = get_user_config()
config_dir = pathlib.Path(config_obj['replay_dir'])
src_path = config_dir / f'{template_name}.json'
yml_path = f'.{template_name}.yml' # relative to root of generated project
logger.debug(f"Reading cookiecutter replay data from {src_path}")
with open(src_path) as f:
cookiecutter_opts = json.load(f)
yaml_opts = {k:v
for k,v in sorted(cookiecutter_opts['cookiecutter'].items())
if not k.startswith('_')}
yaml = YAML()
yaml.default_flow_style=False
yaml.width=4096
yaml.indent(offset=4)
logger.debug(f"Dumping cookiecutter replay (YAML) info to {yml_path}")
with open(yml_path, 'w') as fw:
yaml.dump({'default_context': yaml_opts}, fw)
def write_to_global_results(build_status_dict):
if isfile(GLOBAL_BUILD_RESULTS_PATH):
fp = open(GLOBAL_BUILD_RESULTS_PATH, "r+")
yaml_dict = YAML().load(fp)
else:
fp = open(GLOBAL_BUILD_RESULTS_PATH, "w")
yaml_dict = dict()
for pkg_name, info_dict in build_status_dict.items():
pkg_dict = yaml_dict.setdefault(pkg_name, {})
buildtest_number = len(pkg_dict) + 1
pkg_dict[buildtest_number] = info_dict
fp.seek(0)
YAML().dump(yaml_dict, fp)
fp.truncate()
fp.close()
def get_sdc_env(conda_activate, sdc_src, sdc_recipe, python, numpy, channels):
def create_env_list(packages, exclude=''):
env_list = []
env_set = set()
for item in packages:
package = re.search(r"[\w-]+", item).group()
version = ''
if re.search(r"\d+\.[\d\*]*\.?[\d\*]*",
item) and '<=' not in item and '>=' not in item:
version = '={}'.format(
re.search(r"\d+\.[\d\*]*\.?[\d\*]*", item).group())
if package not in env_set and package not in exclude:
env_set.add(package)
env_list.append(f'{package}{version}')
return env_list
from ruamel_yaml import YAML
yaml = YAML()
sdc_recipe_render = os.path.join(sdc_src, 'sdc_recipe_render.yaml')
# Create environment with conda-build
sdc_render_env = 'sdc_render'
sdc_render_env_activate = get_activate_env_cmd(conda_activate,
sdc_render_env)
format_print('Render sdc build and test environment using conda-build')
create_conda_env(conda_activate, sdc_render_env, python, ['conda-build'])
run_command('{} && {}'.format(
sdc_render_env_activate, ' '.join([
f'conda render --python={python}', f'--numpy={numpy}',
f'{channels} -f {sdc_recipe_render} {sdc_recipe}'
])))
with open(sdc_recipe_render, 'r') as recipe:
data = yaml.load(recipe)
build = data['requirements']['build']
host = data['requirements']['host']
run = data['requirements']['run']
test = data['test']['requires']
return {
'build': create_env_list(build + host + run, 'vs2017_win-64'),
'test': create_env_list(run + test)
}
def get_all_tags():
all_tags = []
yaml = YAML()
for dir in POSTS_DIR:
path = get_path(dir)
for file in glob.glob(os.path.join(path, '*.md')):
meta = yaml.load(get_yaml(file)[0])
if 'tags' in meta:
for tag in meta['tags']:
if tag not in all_tags:
all_tags.append(tag)
else:
raise Exception("Cannot found 'tags' in \
post '{}' !".format(file))
return all_tags
def __init__(self,
path: str,
file_path_yml: Optional[str] = None,
loglevel=logging.DEBUG):
"""
Parameters
----------
path
location where the directory/txt folder to the data can be found.
file_path_yml
path to the yaml config file
loglevel
level of the logger
"""
self.logger = get_logger(self.__class__.__name__, loglevel=loglevel)
# create a yaml file reader
self.yaml = YAML()
self.yaml.indent(mapping=2, sequence=4, offset=2)
# self.yaml.indent(offset=2)
self.yaml.default_flow_style = False
self.yaml.width = 4096
# attributes that change upon changing the starting dir
#: configurations for the run. also saved configurations for the reader under the respective reader name
self.configs = {}
self.reader_data = {}
self.interesting_proteins, self.go_analysis_gene_names = None, None
# properties
self._start_dir = None
self._file_path_yaml = None
# list to store all selectable terms; custom and provided
self.list_full_gos = []
self.list_full_pathways = []
# set the specified dirs
self.start_dir = path
if file_path_yml is not None:
self.file_path_yaml = file_path_yml
def test__default_mirror_configs_jinja(tmpdir):
"""test jinja parsing is correct/valid"""
ll = "DEBUG"
mirror_dir = tmpdir.mkdir("mirror_dir")
configs = _default_mirror_configs(Path(mirror_dir), ll)
for conf in configs:
assert conf.exists()
with open(conf, 'r') as fp:
yaml = YAML().load(fp)
assert yaml["mirror_dir"].startswith(mirror_dir.strpath)
assert yaml["log_dir"].startswith(mirror_dir.strpath)
assert yaml["log_level"] == ll
def yaml_loader(request):
if request.param == "ruamel_yaml":
try:
from ruamel_yaml import YAML # type: ignore
yaml_loader = YAML().load
except ImportError:
from ruamel.yaml import YAML # type: ignore
yaml_loader = YAML().load
module_names = ("ruamel_yaml", "ruamel.yaml")
else:
import yaml # type: ignore
def yaml_loader(x):
return yaml.load(x, Loader=yaml.FullLoader)
module_names = ("pyyaml", )
old_props = serial.YAML_MODULE_PRIORITIES
serial.YAML_MODULE_PRIORITIES = module_names
yield yaml_loader
serial.YAML_MODULE_PRIORITIES = old_props
def __init__(self, processID, grid):
self.processID = processID
self.target = grid.reward
self.threadName = "PO-SMC-" + grid.name if SMC else "PO-UCT" + grid.name
solver = POMCPV
yaml = YAML()
with open('exp_param.yml', 'r') as param:
args = yaml.load(param)
res = MapStat(grid, confidence_interval=2.0)
# args['n_start_states'] = res["particles"]
args['min_particle_count'] = res["particles"]
res = MapStat(grid, confidence_interval=1.0)
args['max_particle_count'] = res["particles"]
args['n_start_states'] = res["particles"]
print("POPULATION SIZE %d SAMPLE SIZE: %d" %
(res["num_states"], res["particles"]))
# np.random.seed(args['seed'])
# random.seed(a=args['seed'])
env = RobotModel(args, grid)
self.agent = AgentSMC(env, solver)
def load_about_md(path):
"""Load contents of ``about.md`` file with title and subtitle from ``path``."""
logger.info("Loading information from about.md at %s", path)
with open(path, "rt") as inputf:
header = []
lines = [line.rstrip() for line in inputf.readlines()]
# Load meta data, if any.
if lines and lines[0] and lines[0].startswith("----"):
for line in lines[1:]:
if line.startswith("----"):
break
else:
header.append(line)
lines = lines[len(header) + 2:]
# Get title and short title, finally create About object.
meta = YAML().load("\n".join(header)) or {}
title = meta.get("title", "Untitled")
short_title = meta.get("short_title", title or "untitled")
readme = "\n".join([line.rstrip() for line in lines]).lstrip()
return About(title=title, short_title=short_title, readme=readme)
def get_categories():
all_categories = []
yaml = YAML()
for dir in POSTS_DIR:
path = get_path(dir)
for file in glob.glob(os.path.join(path, '*.md')):
meta = yaml.load(get_yaml(file)[0])
if 'category' in meta:
if type(meta['category']) == list:
err_msg = (
"[Error] File {} 'category' type"
" can not be LIST!").format(file)
raise Exception(err_msg)
else:
if meta['category'] not in all_categories:
all_categories.append(meta['category'])
else:
if 'categories' in meta:
if type(meta['categories']) == str:
error_msg = (
"[Error] File {} 'categories' type"
" can not be STR!").format(file)
raise Exception(error_msg)
for ctg in meta['categories']:
if ctg not in all_categories:
all_categories.append(ctg)
else:
err_msg = (
"[Error] File:{} at least "
"have one category.").format(file)
print(err_msg)
return all_categories
def _before_sub_resource(self):
try:
if self._have_resource_value():
resource_value = self._get_one_resource_value()
client = KubernetesClient(
os.path.join(settings.OMC_KUBE_CONFIG_DIR, resource_value,
'config'))
else:
client = KubernetesClient()
self.context['common'] = {'client': client}
except Exception as e:
# some action no need to create load config, get config action e.g.
raise Exception('invalid kubenetes config')
if __name__ == '__main__':
client = KubernetesClient()
ret = client.list_service_for_all_namespaces(watch=False)
yaml = YAML()
with open('/Users/luganlin/.omc/config/kube/cd150/config') as f:
content = yaml.load(f)
print(ObjectUtils.get_node(content, "clusters[0].cluster.server"))
# print(client.read_namespaced_pod("postgres-svc-5685d4bc7-l6j4m", 'default'))
# print(client.read_namespaced_pod_template("postgres-svc-5685d4bc7-l6j4m", 'default'))
# print(client.read_namspaced_event("postgres-svc-5685d4bc7-l6j4m", 'default'))
console.log(ret)
# for i in ret.items:
# print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
class MSPInitializer:
# set all file names that are required
yml_file_name_tmp = "config_tmp.yml"
yml_file_name = "config.yml"
default_yml_name = "ms_analysis_default.yml"
go_path = "go_terms"
pathway_path = "pathways"
possible_gos = sorted([
x for x in os.listdir(os.path.join(path_package_config, go_path))
if x.endswith(".txt")
])
possible_pathways = sorted([
x for x in os.listdir(os.path.join(path_package_config, pathway_path))
if x.endswith(".txt")
])
def __init__(self,
dir_: str,
file_path_yml: Optional[str] = None,
loglevel=logging.DEBUG):
self.logger = get_logger(self.__class__.__name__, loglevel=loglevel)
# create a yaml file reader
self.yaml = YAML()
# self.yaml.indent(mapping=2, sequence=4, offset=2)
self.yaml.indent(offset=2)
self.yaml.default_flow_style = False
self.yaml.width = 4096
# attributes that change upon changing the starting dir
self.configs = {}
self.reader_data = {}
self.interesting_proteins, self.go_analysis_gene_names = None, None
# properties
self._start_dir = None
self._file_path_yaml = None
# set the specified dirs
self.start_dir = dir_
if file_path_yml is not None:
self.file_path_yaml = file_path_yml
@property
def start_dir(self):
return self._start_dir
@start_dir.setter
def start_dir(self, start_dir):
start_dir = os.path.normpath(start_dir)
# make sure to be on the right level and set starting dir
if os.path.split(start_dir)[1] == "txt":
self.logger.debug("Removing txt ending from path")
self._start_dir = os.path.split(start_dir)[0]
else:
self._start_dir = start_dir
self.logger.info(f"Starting dir: {self.start_dir}")
# set all attributes back None that where file specific
self.configs = {}
self.reader_data = {}
self.file_path_yaml = "file"
@property
def path_config(self):
return os.path.join(self.start_dir, "config")
@property
def file_path_yaml(self):
return self._file_path_yaml
@file_path_yaml.setter
def file_path_yaml(self, file_path_yml: str):
"""
Parameters
----------
file_path_yml
can be either:
- "default"
- "file"
- a path to a yml file
Raises
------
ValueError
if no valid value was provided
FileNotFoundError
if the file specified by the file_path_yml was not found
"""
if file_path_yml.lower() == "default":
self._file_path_yaml = self.get_default_yml_path()
elif file_path_yml.lower() == "file":
if self.has_yml_file():
self._file_path_yaml = os.path.join(
self.start_dir, "config", MSPInitializer.yml_file_name)
else:
self._file_path_yaml = self.get_default_yml_path()
elif file_path_yml.lower().endswith(('.yml', '.yaml')):
self._file_path_yaml = os.path.normpath(file_path_yml)
else:
raise ValueError(
f"Invalid value provided for yaml file: {file_path_yml}")
self.logger.debug("yml file location: %s", self._file_path_yaml)
# load the config from the yml file
self.logger.info("loading yml file")
with open(self.file_path_yaml) as f:
self.configs = self.yaml.load(f)
self.logger.debug(f"Config file contents: {self.configs}")
def init_config(self):
"""
Creates the directory to save the configuration file and saves the configuration
"""
os.makedirs(self.path_config, exist_ok=True)
self.update_config_file()
def has_yml_file(self) -> bool:
if not os.path.isdir(self.start_dir):
return False
if "config" in os.listdir(self.start_dir):
self.logger.debug("Found config dir")
config_dir = os.path.join(self.start_dir, "config")
if MSPInitializer.yml_file_name in os.listdir(config_dir):
self.logger.debug("Found config.yml file in config dir")
return True
return False
def get_default_yml_path(self) -> str:
self.logger.debug(
"Loading default yml file from: %s, since no (valid) file was selected",
path_package)
return os.path.join(path_package_config,
MSPInitializer.default_yml_name)
def init_interest_from_txt(
self) -> Tuple[Dict[str, list], Dict[str, list]]:
dict_pathway = {}
dict_go = {}
for pathway in self.configs.get("pathways"):
name, proteins = self.read_config_txt_file(
MSPInitializer.pathway_path, pathway)
dict_pathway[name] = proteins
for go in self.configs.get("go_terms"):
name, proteins = self.read_config_txt_file(MSPInitializer.go_path,
go)
dict_go[name] = proteins
return dict_pathway, dict_go
def read_config_txt_file(self, path, file) -> Tuple[str, list]:
fullpath = os.path.join(path_package_config, path, file)
if path == MSPInitializer.pathway_path:
with open(fullpath) as f:
name = f.readline().strip()
f.readline()
proteins = []
for line in f:
proteins.append(line.strip())
elif path == MSPInitializer.go_path:
name = file.replace(".txt", "")
with open(fullpath) as f:
proteins = []
for line in f:
proteins.append(line.strip())
else:
raise ValueError(f"Invalid path: {path}")
return name, proteins
def update_config_file(self):
# store the config file as tmp
self.logger.debug("Updating yml settings file")
yml_file_loc_tmp = os.path.join(self.path_config,
MSPInitializer.yml_file_name_tmp)
with open(yml_file_loc_tmp, "w") as outfile:
self.yaml.dump(self.configs, outfile)
# delete non tmp if exists
yml_file_loc = os.path.join(self.path_config,
MSPInitializer.yml_file_name)
if MSPInitializer.yml_file_name in os.listdir(self.path_config):
os.remove(yml_file_loc)
# rename to non tmp
os.rename(yml_file_loc_tmp, yml_file_loc)
def read_data(self):
for Reader in BaseReader.__subclasses__():
Reader: Type[BaseReader] # for IDE hints
try:
reader = Reader(self.start_dir,
self.configs.get(Reader.name, {}))
self.configs[str(Reader.name)] = deepcopy(reader.reader_config)
self.reader_data[Reader.name] = reader.full_data
except MissingFilesException:
self.logger.debug("No files found for reader: %s", Reader.name)
# read all proteins and receptors of interest from the config dir
self.logger.info("Reading proteins and receptors of interest")
self.interesting_proteins, self.go_analysis_gene_names = self.init_interest_from_txt(
)
self.update_config_file()
map_index = lambda pos: pos.i * y_size + pos.j
start_new_timer = ModelChecker.check_safe_reachability(
map(map_index, traj), map(map_index,
self.model.obstacle_positions),
map(map_index, self.model.goal_positions))
# STEP log writing
LOGFILE["reward"].append(reward)
LOGFILE["time"].append(elapsed_time)
LOGFILE["sat"].append(start_new_timer)
dtf = pd.DataFrame(LOGFILE)
dtf.to_csv(EXP_LOG + 'map-14-4-6_3000' + '.csv')
if __name__ == '__main__':
yaml = YAML()
with open('exp_param.yml', 'r') as param:
args = yaml.load(param)
init_logger()
np.random.seed(args['seed'])
random.seed(a=args['seed'])
solver = POMCPV
env = RobotModel(args)
agent = AgentSMC(env, solver)
EXP_LOG = "/home/experiments/LargeMap/"
if (not os.path.exists(EXP_LOG)):
os.makedirs(EXP_LOG)
dicounted_return(agent)
