def conda_create_command(
requirement_files, env_name='calliope',
python_version='3', channels=[], ignore=[], run=False):
"""
Parameters
----------
requirements_files : list
env_name : str, optional
python_version: str, optional
channels: list, optional
ignore : list, optional
run : bool, optional
Returns
-------
cmd : str
"""
chan_set = set()
dep_set = set()
for req in requirement_files:
with open(req) as f:
y = yaml.safe_load(f)
# Filter the optional `pip` dict in dependencies
str_deps = list(filter(
lambda i: isinstance(i, str),
y['dependencies']
))
if ignore:
# Filter ignored dependencies
str_deps = list(filter(
lambda i: not any([ign in i for ign in ignore]),
str_deps
))
dep_set.update(str_deps)
chan_set.update(y['channels'])
dep_string = ' '.join(['"{}"'.format(i) for i in sorted(dep_set)])
if channels:
chan_string = ' '.join(['-c ' + i for i in channels])
else:
chan_string = ' '.join(['-c ' + i for i in chan_set])
cmd = 'conda create --name {name} --override-channels {chans} "python={py}" {deps}'.format(
name=env_name,
chans=chan_string,
deps=dep_string,
py=python_version
)
return cmd
def load_yaml(self, yaml_file="example.yaml"):
if not yaml_file:
return None
yaml_data = None
with open(yaml_file, 'r') as fp:
try:
yaml_data = yaml.safe_load(fp)
except yaml.YAMLError as exc:
print(exc)
return yaml_data
def test_object_attrs(setup_teardown_file):
obj = setup_teardown_file[3].create_dataset("test_object", shape=(1,), dtype=float)
assert isinstance(obj.attrs, Attribute)
assert obj.attrs.mode.value == 1
obj.attrs = "test value"
with (setup_teardown_file[1] / "test_object" / ATTRIBUTES_FILENAME).open("r", encoding="utf-8") as meta_file:
meta_data = yaml.safe_load(meta_file)
assert meta_data == "test value"
def conda_create_command(requirement_files,
env_name='calliope',
python_version='3',
channels=[],
ignore=[],
run=False):
"""
Parameters
----------
requirements_files : list
env_name : str, optional
python_version: str, optional
channels: list, optional
ignore : list, optional
run : bool, optional
Returns
-------
cmd : str
"""
chan_set = set()
dep_set = set()
for req in requirement_files:
with open(req) as f:
y = yaml.safe_load(f)
# Filter the optional `pip` dict in dependencies
str_deps = list(filter(lambda i: isinstance(i, str),
y['dependencies']))
if ignore:
# Filter ignored dependencies
str_deps = list(
filter(lambda i: not any([ign in i for ign in ignore]),
str_deps))
dep_set.update(str_deps)
chan_set.update(y['channels'])
dep_string = ' '.join(['"{}"'.format(i) for i in sorted(dep_set)])
if channels:
chan_string = ' '.join(['-c ' + i for i in channels])
else:
chan_string = ' '.join(['-c ' + i for i in chan_set])
cmd = 'conda create --name {name} --override-channels {chans} "python={py}" {deps}'.format(
name=env_name, chans=chan_string, deps=dep_string, py=python_version)
return cmd
def load_yaml(yml_file):
# Load YAML data from file using the "safe" loading option.
try:
yaml_ = yaml.YAML(typ="safe")
with open(yml_file, "rt", encoding="utf-8") as stream:
return yaml_.load(stream)
except yaml.constructor.ConstructorError:
with open(yml_file, "rt", encoding="utf-8") as stream:
# Ensure that the loader remains backward-compatible with legacy
# ruamel.yaml versions (prior to 0.17.0).
return yaml.safe_load(stream)
def upload_package(feedstock_root, recipe_root, config_file):
specific_config = safe_load(open(config_file))
if "channel_targets" in specific_config:
channels = [c.strip().split(" ") for c in specific_config["channel_targets"]]
else:
update_global_config(feedstock_root)
channels = _global_config["targets"]
from .upload_or_check_non_existence import upload_or_check
for owner, channel in channels:
upload_or_check(recipe_root, owner, channel, [config_file])
def build_all(recipes_dir, arch):
folders = os.listdir(recipes_dir)
old_comp_folders = []
new_comp_folders = []
if not folders:
print("Found no recipes to build")
return
if get_host_platform() == "win":
new_comp_folders.extend(folders)
else:
for folder in folders:
built = False
cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
if os.path.exists(cbc):
with open(cbc, "r") as f:
text = ''.join(f.readlines())
if 'channel_sources' in text:
specific_config = safe_load(text)
if "channel_targets" not in specific_config:
raise RuntimeError(
"channel_targets not found in {}".format(folder))
if "channel_sources" in specific_config:
for row in specific_config["channel_sources"]:
channels = [c.strip() for c in row.split(",")]
if channels != ['conda-forge', 'defaults'] and \
channels != ['conda-forge/label/cf201901', 'defaults']:
print(
"Not a standard configuration of channel_sources. Building {} individually."
.format(folder))
conda_build.api.build(
[os.path.join(recipes_dir, folder)],
config=get_config(arch, channels))
built = True
break
if not built:
old_comp_folders.append(folder)
continue
new_comp_folders.append(folder)
if old_comp_folders:
print("Building {} with conda-forge/label/cf201901".format(
','.join(old_comp_folders)))
channel_urls = ['local', 'conda-forge/label/cf201901', 'defaults']
build_folders(recipes_dir, old_comp_folders, arch, channel_urls)
if new_comp_folders:
print("Building {} with conda-forge/label/main".format(
','.join(new_comp_folders)))
channel_urls = ['local', 'conda-forge', 'defaults']
build_folders(recipes_dir, new_comp_folders, arch, channel_urls)
def make_dict_from_meta_file(path):
try:
document, compilers_in_build = dynamic_jinja_to_static_ruamel_yaml(
path)
recipe_dict = ruamel_yaml.safe_load(document)
# add compilers to the dict
for comp in compilers_in_build:
recipe_dict["requirements"]["build"].append(comp)
return recipe_dict
except ruamel_yaml.YAMLError as exc:
print(
"Couldn't create a dictionary from the meta.ruamel_yaml. Error: {}"
.format(exc))
def build_all(recipes_dir, arch):
folders = os.listdir(recipes_dir)
old_comp_folders = []
new_comp_folders = []
if not folders:
print("Found no recipes to build")
return
if get_host_platform() == "win":
old_comp_folders.extend(folders)
else:
for folder in folders:
built = False
cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
if os.path.exists(cbc):
with open(cbc, "r") as f:
text = ''.join(f.readlines())
if 'channel_sources' in text:
specific_config = safe_load(text)
if "channel_targets" not in specific_config:
raise RuntimeError("channel_targets not found in {}".format(folder))
if "channel_sources" in specific_config:
for row in specific_config["channel_sources"]:
channels = [c.strip() for c in row.split(",")]
if channels != ['conda-forge/label/gcc7', 'defaults'] and \
channels != ['conda-forge', 'defaults']:
print("Not a standard configuration of channel_sources. Building {} individually.".format(folder))
conda_build.api.build([os.path.join(recipes_dir, folder)], config=get_config(arch, channels))
built = True
break
if not built:
new_comp_folders.append(folder)
continue
old_comp_folders.append(folder)
if old_comp_folders:
print("Building {} with conda-forge/label/main".format(','.join(old_comp_folders)))
if new_comp_folders:
print("Building {} with conda-forge/label/gcc7".format(','.join(new_comp_folders)))
if old_comp_folders:
channel_urls = ['local', 'conda-forge', 'defaults']
build_folders(recipes_dir, old_comp_folders, arch, channel_urls)
if new_comp_folders:
print("Building {} with conda-forge/label/gcc7".format(','.join(old_comp_folders)))
channel_urls = ['local', 'conda-forge/label/gcc7', 'defaults']
build_folders(recipes_dir, new_comp_folders, arch, channel_urls)
def setup_conda_rc(feedstock_root, recipe_root, config_file):
specific_config = safe_load(open(config_file))
if "channel_sources" in specific_config:
# Due to rendering we may have more than one row for channel_sources
# if nothing gets zipped with it
first_row = specific_config["channel_sources"][0] # type: str
channels = [c.strip() for c in first_row.split(",")]
else:
update_global_config(feedstock_root)
channels = _global_config["sources"]
call(["conda", "config", "--remove", "channels", "defaults"])
for c in reversed(channels):
call(["conda", "config", "--add", "channels", c])
call(["conda", "config", "--set", "show_channel_urls", "true"])
def load_md(src):
"""Load metadata in YAML from a source.
Arguments:
src: dict-like (data), string (filename), or file-like
"""
if isinstance(src, collections.Mapping): # dict or yaml is given
rawconfig = src
elif isinstance(src, str): # filename is given
with open(src, "r") as f:
rawconfig = yaml.load(f)
elif (isinstance(src, TextIOBase)): # file-like is given
#rawconfig = yaml.load(src) # deprecated/unsafe
rawconfig = yaml.safe_load(src)
return parse_md(rawconfig)
def is_nonraw_object_directory(directory):
meta_filename = directory / META_FILENAME
if not meta_filename.exists():
return False
with meta_filename.open("r", encoding="utf-8") as meta_file:
meta_data = yaml.safe_load(meta_file)
if not isinstance(meta_data, dict):
return False
if EXDIR_METANAME not in meta_data:
return False
if TYPE_METANAME not in meta_data[EXDIR_METANAME]:
return False
valid_types = [DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME]
if meta_data[EXDIR_METANAME][TYPE_METANAME] not in valid_types:
return False
return True
def fail_if_travis_not_allowed_for_arch(config_file):
specific_config = safe_load(open(config_file))
if "channel_targets" in specific_config:
channels = [c.strip().split(" ") for c in specific_config["channel_targets"]]
else:
update_global_config(feedstock_root)
channels = _global_config["channels"]["targets"]
upload_to_conda_forge = any(owner == "conda-forge" for owner, _ in channels)
if (
upload_to_conda_forge
and os.environ.get("CI", None) == "travis"
and (
platform.uname().machine.lower() in ["x86_64", "amd64"]
or platform.system().lower() != "linux"
)
):
raise RuntimeError("Travis CI cannot be used on x86_64 in conda-forge!")
def lock(flow, pf, py, lab):
output = P.ENVENTURES[flow, pf, py, lab]
if not output.parent.exists():
output.parent.mkdir(parents=True)
composite = {"name": output.name, CHN: [], DEP: []}
for env in P.ENV_DEPS[flow, pf, py, lab]:
composite = merge(composite, safe_load(env.read_text()))
print(safe_dump(composite, default_flow_style=False), flush=True)
with tempfile.TemporaryDirectory() as td:
tdp = Path(td)
env = tdp / "environment.yml"
env.write_text(safe_dump(composite, default_flow_style=False))
args = [P.CONDA_EXE, "lock", "--platform", pf]
subprocess.check_call(args, cwd=td)
if not output.parent.exists():
output.parent.mkdir(parents=True)
output.write_text((tdp / f"conda-{pf}.lock").read_text())
return 0
def test_yaml_surface(self):
gas = ct.Solution('ptcombust.yaml', 'gas')
surf = ct.Interface('ptcombust.yaml', 'Pt_surf', [gas])
gas.TPY = 900, ct.one_atm, np.ones(gas.n_species)
surf.coverages = np.ones(surf.n_species)
surf.write_yaml('ptcombust-generated.yaml')
with open('ptcombust-generated.yaml') as infile:
generated = yaml.safe_load(infile)
for key in ('phases', 'species', 'gas-reactions', 'Pt_surf-reactions'):
self.assertIn(key, generated)
self.assertEqual(len(generated['gas-reactions']), gas.n_reactions)
self.assertEqual(len(generated['Pt_surf-reactions']), surf.n_reactions)
self.assertEqual(len(generated['species']), surf.n_total_species)
gas2 = ct.Solution('ptcombust-generated.yaml', 'gas')
surf2 = ct.Solution('ptcombust-generated.yaml', 'Pt_surf', [gas2])
self.assertArrayNear(surf.concentrations, surf2.concentrations)
self.assertArrayNear(surf.partial_molar_enthalpies,
surf2.partial_molar_enthalpies)
self.assertArrayNear(surf.forward_rate_constants,
surf2.forward_rate_constants)
def fail_if_outdated_windows_ci(feedstock_root):
if sys.platform != "win32":
return
if "APPVEYOR_ACCOUNT_NAME" in os.environ:
provider = "appveyor"
if os.environ["APPVEYOR_ACCOUNT_NAME"] != "conda-forge":
return
if "APPVEYOR_PULL_REQUEST_NUMBER" not in os.environ:
return
elif "BUILD_REPOSITORY_NAME" in os.environ:
provider = "azure"
if not os.environ["BUILD_REPOSITORY_NAME"].startswith("conda-forge/"):
return
if "SYSTEM_PULLREQUEST_PULLREQUESTID" not in os.environ:
return
else:
return
with open(os.path.join(feedstock_root, "conda-forge.yml")) as f:
config = safe_load(f)
if "provider" in config and "win" in config["provider"]:
provider_cfg = config["provider"]["win"]
if provider_cfg != "azure":
return
if provider == "appveyor":
raise RuntimeError(
"This PR needs a rerender to switch from appveyor to azure")
if (
provider == "azure"
and (
os.getenv("UPLOAD_PACKAGES", "False") == "False"
or os.path.exists(".appveyor.yml")
)
):
raise RuntimeError(
"This PR needs a rerender to switch from appveyor to azure")
def test_create_object_directory(setup_teardown_folder):
with pytest.raises(ValueError):
exob._create_object_directory(pathlib.Path(setup_teardown_folder[2]), exob._default_metadata("wrong_typename"))
exob._create_object_directory(pathlib.Path(setup_teardown_folder[2]), exob._default_metadata(exob.DATASET_TYPENAME))
assert setup_teardown_folder[2].is_dir()
file_path = setup_teardown_folder[2] / exob.META_FILENAME
assert file_path.is_file()
compare_metadata = {
exob.EXDIR_METANAME: {
exob.TYPE_METANAME: exob.DATASET_TYPENAME,
exob.VERSION_METANAME: 1}
}
with file_path.open("r", encoding="utf-8") as meta_file:
metadata = yaml.safe_load(meta_file)
assert metadata == compare_metadata
with pytest.raises(IOError):
exob._create_object_directory(pathlib.Path(setup_teardown_folder[2]), exob.DATASET_TYPENAME)
def upload_package(feedstock_root, recipe_root, config_file, validate, private, feedstock_name):
if feedstock_name is None and validate:
raise RuntimeError("You must supply the --feedstock-name option if validating!")
specific_config = safe_load(open(config_file))
if "channel_targets" in specific_config:
channels = [c.strip().split(" ") for c in specific_config["channel_targets"]]
source_channels = ",".join(
[c.strip() for c in specific_config["channel_sources"]])
else:
update_global_config(feedstock_root)
channels = _global_config["channels"]["targets"]
source_channels = ",".join(_global_config["channels"]["sources"])
if "UPLOAD_ON_BRANCH" in os.environ:
if "GIT_BRANCH" not in os.environ:
print(
"WARNING: UPLOAD_ON_BRANCH env variable set, "
"but GIT_BRANCH not set. Skipping check")
else:
if os.environ["UPLOAD_ON_BRANCH"] != os.environ["GIT_BRANCH"]:
print(
"The branch {} is not configured to be "
"uploaded".format(os.environ["GIT_BRANCH"]))
return
upload_to_conda_forge = any(owner == "conda-forge" for owner, _ in channels)
if upload_to_conda_forge and "channel_sources" in specific_config:
allowed_channels = [
"conda-forge", "conda-forge/label/", "defaults", "c4aarch64",
"c4armv7l"]
for source_channel in source_channels.split(","):
if source_channel.startswith('https://conda-web.anaconda.org/'):
source_channel = source_channel[len('https://conda-web.anaconda.org/'):]
for c in allowed_channels:
if source_channel.startswith(c):
break
else:
print(
"Uploading to %s with source channel '%s' "
"is not allowed" % ("conda-forge", source_channel))
return
# get the git sha of the current commit
git_sha = subprocess.run(
"git rev-parse HEAD",
check=True,
stdout=subprocess.PIPE,
shell=True,
cwd=feedstock_root,
).stdout.decode("utf-8").strip()
if len(git_sha) == 0:
git_sha = None
print("Did not find git SHA for this build!")
else:
print("Found git SHA %s for this build!" % git_sha)
for owner, channel in channels:
if validate and owner == "conda-forge":
retry_upload_or_check(
feedstock_name, recipe_root, STAGING, channel,
[config_file], validate=True, git_sha=git_sha)
else:
retry_upload_or_check(
feedstock_name, recipe_root, owner, channel,
[config_file], validate=False, private_upload=private)
import sys
if sys.version_info < (3, 0):
sys.stdout.write("Sorry, Python 3 og higher required\n")
sys.exit(1)
import os, json, urllib.request
import ruamel_yaml as yaml
if len(sys.argv) != 2:
print("usage: {} music.yml".format(sys.argv[0]))
sys.exit(1)
filename = sys.argv[1]
with open(filename) as f:
sidlist = yaml.safe_load(f)
server = sidlist['server'] + '/'
dstdir = 'sids/'
if not os.path.exists(dstdir):
os.makedirs(dstdir)
for i in sidlist['songs']:
url = server + i['file']
basename = os.path.basename(url)
dstfile = dstdir + basename
i['file'] = dstfile
if not os.path.isfile(dstfile):
print('retrieving', dstfile)
urllib.request.urlretrieve(url, dstfile)
# save playlist to json
out = json.dumps(sidlist, indent=2)
root_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), os.path.pardir))
# import the osgameclones data
osgc_path = os.path.realpath(
os.path.join(root_path, os.path.pardir, '11_osgameclones.git',
'games'))
files = os.listdir(osgc_path)
# iterate over all yaml files in osgameclones/data folder
osgc_entries = []
for file in files:
# read yaml
with open(os.path.join(osgc_path, file), 'r') as stream:
try:
_ = yaml.safe_load(stream)
except yaml.YAMLError as exc:
raise exc
# add to entries
osgc_entries.extend(_)
print('{} entries in osgameclones'.format(len(osgc_entries)))
# eliminate the ignored entries
osgc_entries = [
x for x in osgc_entries if x['name'] not in osgc_ignored_entries
]
# fix names and licenses (so they are not longer detected as deviations downstreams)
for index, entry in enumerate(osgc_entries):
name = entry['name']
def test_cli_yaml(script_runner):
"""is the default output valid yaml?"""
ret = script_runner.run("jupyter", "starters", "list")
assert ret.success
assert safe_load(ret.stdout)
def load_configuration(filepath):
with open(filepath, "r") as f:
config = yaml.safe_load(f)
return config
def _open_or_create(self):
attrs = {}
if self.filename.exists(): # NOTE str for Python 3.5 support
with self.filename.open("r", encoding="utf-8") as meta_file:
attrs = yaml.safe_load(meta_file)
return attrs
def configure(self, config_file):
stream = open(config_file, 'r')
config = yaml.safe_load(stream)
self.name = config['name']
self.re_share = config['re_share']
self.imp_share = config['re_share']
self.nuc_share = config['nuc_share']
if self.log_level >= 1:
print("Loading cell: {0}".format(self.name))
# Sets an order of importance for typical days to approximate the different inputs
self.weight = config['weight']
# Set consumption
if self.log_level == 2: print('\tSet consumption,', end='')
self.cons = np.loadtxt(open(config['consumption_file'], 'r'),
skiprows=1)
self.cons = self.cons.reshape(-1, self.dt).mean(
axis=1) # Reshape according to the time step
self.cons = self.cons * 1e6
# Producers
self.wt_parks = []
self.pv_parks = []
## PV
if self.log_level == 2: print('Set PV parks,', end='')
if 'regions' in config['pv']:
for i in range(len(config['pv']['regions'])):
pvpark = PVPark(config['pv']['regions'][i], config['pv'],
self.size, self.dt)
self.pv_parks.append(pvpark)
else:
pvpark = PVPark(config['pv'], config['pv'], self.size, self.dt)
self.pv_parks.append(pvpark)
## WT
if self.log_level == 2: print('Set WT parks,', end='')
for onoff in config['wind']:
if 'regions' in config['wind'][onoff]:
for i in range(len(config['wind'][onoff]['regions'])):
wtpark = WTPark(config['wind'][onoff]['regions'][i],
config['wind'][onoff], self.size, self.dt,
onoff == 'onshore')
self.wt_parks.append(wtpark)
else:
wtpark = WTPark(config['wind'][onoff], config['wind'][onoff],
self.size, self.dt, onoff == 'onshore')
self.wt_parks.append(wtpark)
## CCGT
if self.log_level == 2: print('Set CCGT,', end='')
self.ccgt = CCGTPlant(config['ccgt'])
## Coal
if self.log_level == 2: print('Set Coal,', end='')
self.coal = CoalPlant(config['coal'])
## Nuclear
if self.log_level == 2: print('Set Nuclear,', end='')
self.nuclear = NuclearPlant(config['nuclear'], self.size)
## Hydro Dam
if self.log_level == 2: print('Set Dam,', end='')
self.dam = DamPlant(config['hydro_dam'], self.size, self.dt)
## Run-of-the-river
if self.log_level == 2: print('Set Run-of-the-river,', end='')
self.runriver = RunRiver(config['hydro_river'], self.size, self.dt)
# Storage
self.bat_parks = []
## Batteries
if self.log_level == 2: print('Set Battery parks,', end='')
batt = Battery(config['battery'], "Lithium", self.size, self.dt)
self.bat_parks.append(batt)
## PHES
if self.log_level == 2: print('Set PHES,', end='')
self.phes = PHES(config['phes'], self.size)
## PtG
if self.log_level == 2: print('Set Power-to-Gas,', end='')
self.ptg = PtG(config['ptg'], self.size, self.dt)
## Gas
if self.log_level == 2: print('Set Gas storage')
self.gas = Gas(config['gas'], self.size)
def setUpClass(cls):
utilities.CanteraTest.setUpClass()
cls.yml_file = pjoin(cls.test_data_dir, "thermo-models.yaml")
with open(cls.yml_file, 'rt', encoding="utf-8") as stream:
cls.yml = yaml.safe_load(stream)
x, y, w, h = cv2.boundingRect(cnt)
bb.append([x, y, w, h])
#cv2.rectangle(img, (x,y), (x+w, y+h), (255),2)
#cv2.imshow("contours", img)
binary = []
for img in Images:
img = np.array(img)
mask = img == 0
img[mask] = 0
img[np.logical_not(mask)] = 255
binary.append(img)
for metafilename in sorted(glob.glob('./images/*_poses.yaml')):
meta_file = open(metafilename)
meta = yaml.safe_load(meta_file)
t_ = [i * 1000 for i in meta['schaltgabel']['pose'][0]]
t.append(t_)
quaternion_list = meta['schaltgabel']['pose'][1]
modified_quaternion = [
quaternion_list[1], quaternion_list[2], quaternion_list[3],
quaternion_list[0]
]
r.append(R.from_quat(modified_quaternion).as_dcm().flatten().tolist())
#different notation of quaternions between sicpy and labelfusion
#https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.transform.Rotation.from_quat.html
#https://github.com/RobotLocomotion/LabelFusion/issues/41
posedata_file = open('./posedata.yml')
posedata = yaml.safe_load(posedata_file)
def __init__(self, configuration_file, log_level):
"""Create Case object based on the given configuration file.
Parameters
----------
configuration_file: str
Path to the case configuration file (a .yml file)
log_level: int
A value between 0 and 2: 0 for little information, 2 for lots of
information
Returns
-------
object
Case object
"""
self.log_level = log_level
stream = open(configuration_file, 'r')
config = yaml.safe_load(stream)
# === Load parameters ===
self.name = config['name']
self.dt = config['dt']
self.size = int(8760 / self.dt)
self.use_CPLEX = config['use_CPLEX']
self.optiTD = config['optiTD']
if self.log_level >= 1:
print(Style.BRIGHT + "=== Case configuration ===" + Style.NORMAL)
if self.log_level == 2:
print(
"Case name: \t{0}\nTime step: \t{1}\nProblem size: \t{2}\nCPLEX solver: \t{3}\nTypical Days: \t{4}"
.format(self.name, self.dt, self.size, self.use_CPLEX,
self.optiTD))
if self.optiTD: # If Typical Days are used
self.replicate = config['replicate']
self.loadTD = config['loadTD']
self.nrTD = config['nrTD']
if self.log_level == 2:
print("Load TD: \t{0}\nNumber of TD: \t{1}".format(
self.loadTD, self.nrTD))
if not (self.loadTD):
print("Replicate: \t{0}".format(self.replicate))
# Policies
re_share = config['re_share']
imp_share = config['imp_share']
self.policies = [re_share, imp_share]
if self.log_level == 2:
print("Max RE share: \t{0}\nMax imp share: \t{1}".format(
re_share, imp_share))
# Cells
self.cells = []
nbcells = len(config['cells'])
if self.log_level >= 1:
print('\n' + Style.BRIGHT + "=== Cells configuration ===" +
Style.NORMAL)
for cell in config['cells']:
self.cells.append(Cell(self.size, self.dt, self.log_level))
self.cells[-1].configure(cell)
# e_exch, gas_exch, e_imp and gas_imp
e_exch = {}
gas_exch = {}
e_imp = {}
gas_imp = {}
for i, cell in enumerate(self.cells):
e_exch[cell.name] = {}
gas_exch[cell.name] = {}
e_imp[cell.name] = config['e_imp'][i]
gas_imp[cell.name] = config['gas_imp'][i]
for j, neighbour in enumerate(self.cells):
e_exch[cell.name][neighbour.name] = config['e_exch'][i][j]
gas_exch[cell.name][neighbour.name] = config['gas_exch'][i][j]
self.e_exch = e_exch
self.gas_exch = gas_exch
self.e_imp = e_imp
self.gas_imp = gas_imp
self.grid = Grid(self.e_imp, self.e_exch, self.gas_imp, self.gas_exch)
# Computed values
self.E = {}
self.E_tot = {}
self.EROI_pv = {}
self.EROI_wt = {}
self.share = {}
self.share_tot = {}
self.LF = {}
self.E_invest = {}
self.E_invest_tot = {}
# Typical Days
if self.optiTD:
assert self.dt < 12, "Timestep to big to compute typical days."
if self.loadTD: # Load Typical Days from least error file
filename = os.path.join('typical_days', self.name + '_' + \
'TDays'+ '_' + str(self.dt) + '_' + str(self.nrTD) \
+ 'TD_*')
possible_TD = glob.glob(filename)
min_error = 1e9
for f in possible_TD:
error = float(f.split('_')[-1][:-4])
if error < min_error:
TDays_f = f
min_error = error
if self.log_level >= 1:
print("\n" + Style.BRIGHT + "=== Typical Days ===" +
Style.NORMAL)
print('Min error : \t{:.6f}'.format(min_error))
self.TDays = np.loadtxt(TDays_f)
else: # Start multithreaded computation to find the Typical Days
threads = 8
if self.log_level >= 1:
print("\n" + Style.BRIGHT + "=== Typical Days ===" +
Style.NORMAL)
print("Compute typical days with " + str(threads) +
" threads")
manager = multiprocessing.Manager()
return_dict = manager.dict()
jobs = [None] * threads
self.replicate = int(self.replicate / threads)
for i in range(threads): # Start threads
jobs[i] = multiprocessing.Process(target=self.computeTD, \
args=(i, return_dict))
jobs[i].start()
for i in range(threads): # Wait for threads to finish
jobs[i].join()
best_error = 1e6
for (TDays, error) in return_dict.values():
if error < best_error:
self.TDays = TDays.astype('int')
best_error = error
if self.log_level >= 1:
print('Min error : \t{:.6f}'.format(best_error))
dtperday = int(24 / self.dt)
dtpd = int(24 / self.dt)
powerTD = np.zeros(self.size)
# Plot the load duration curve with and without typical days
# for the user to assess the precision
if self.log_level == 2:
for cell in self.cells:
for i in range(0, self.size, dtperday):
TD = self.TDays[i]
powerTD[i:i +
dtpd] = cell.cons[TD * dtpd:TD * dtpd +
dtpd]
sort = np.sort(cell.cons)[::-1]
exceedence = np.arange(1., len(sort) + 1) / len(sort)
l1, = plt.plot(exceedence * 100, sort)
sort = np.sort(powerTD)[::-1]
exceedence = np.arange(1., len(sort) + 1) / len(sort)
l2, = plt.plot(exceedence * 100, sort)
plt.legend((l1, l2), ('Without TD', 'With TD'))
plt.title("LDC with and without Typical Days : " +
cell.name)
plt.show()
filename = os.path.join('typical_days', self.name + '_' + \
'TDays'+ '_' + str(self.dt) + '_' + str(self.nrTD) + \
'TD_' + str(best_error) + '.txt')
np.savetxt(filename, self.TDays, fmt='%d')
def _load_chip_design(self, filename):
with open(filename, "r") as f:
#self.chip_design = yaml.load(f)
self.chip_design = yaml.safe_load(f)
s0 = chain(
orthanc.get_s(dx.oid(), DicomLevel.INSTANCES) | clf.classify_s()
| redis.put_s())()
s1 = chain(
orthanc.get_s(hx.oid(), DicomLevel.INSTANCES) | clf.classify_s()
| redis.put_s())()
s0.get()
s1.get()
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logging.debug("Simple Distributed Diana Test Script")
service_cfg = os.environ.get("DIANA_SERVICES_CFG", "./services.yml")
with open(service_cfg, "r") as f:
services = yaml.safe_load(f)
dcm_dir = "/Users/derek/data/DICOM/airway phantom/DICOM/PA2/ST1/SE1"
files = File(location=dcm_dir)
orthanc = Orthanc(**services['orthanc'])
clf = Classifier()
redis = Redis(**services['redis'])
test_celery()
def make_build_number(feedstock_root, recipe_root, config_file):
"""
General logic
The purpose of this is to ensure that the new compilers have build
numbers > 1000 and legacy compilers have a build number < 1000.
This is done by reading the build_number_decrement which is rendered
into all the recipes.
For linux and osx we want to avoid building for the legacy compilers
with build numbers > 1000
Example matrix
- {'compiler_c': 'toolchain_c', 'build_number_decrement': 1000}
- {'compiler_c': 'gcc', 'build_number_decrement': 0}
"""
specific_config = safe_load(open(config_file))
build_number_dec = int(specific_config.get("build_number_decrement", [0])[0])
if build_number_dec == 0:
return
use_legacy_compilers = False
for key in {"c", "cxx", "fortran"}:
if "toolchain_{}".format(key) in specific_config.get(
'{}_compiler'.format(key), ""):
use_legacy_compilers = True
break
import conda_build.api
rendered_recipe = conda_build.api.render(
recipe_path=recipe_root, variants=specific_config
)
build_numbers = set()
for recipe, _, _ in rendered_recipe:
build_numbers.add(int(recipe.get_value("build/number")))
if len(build_numbers) > 1:
raise ValueError("More than one build number found, giving up")
if len(build_numbers) == 0:
print("> conda-forge:: No build number found. Presuming build string")
return
try:
build_number_int = build_numbers.pop()
if build_number_int < 1000:
if not use_legacy_compilers:
raise ValueError(
"Only legacy compilers only valid with build numbers < 1000"
)
new_build_number = build_number_int
else:
new_build_number = build_number_int - build_number_dec
config_dir, filename = os.path.split(config_file)
with open(os.path.join(config_dir, "clobber_" + filename), "w") as fo:
data = {"build": {"number": new_build_number}}
print("> conda-forge:: Build number clobber {} -> {}".format(
build_number_int, new_build_number))
safe_dump(data, fo)
except ValueError:
# This is a NON string build number
# we have this for things like the blas mutex and a few other similar cases
print("> conda-forge:: No build number clobber gererated!")
import traceback
traceback.print_exc()
try: # ... to read json
i = args.infile.read()
if jinja2:
# additional files can be used with {% include "file" %}
dirs = [os.getcwd(), os.path.dirname(os.path.realpath(__file__)) + "/../top"]
loader = jinja2.FileSystemLoader(dirs)
env = jinja2.Environment(loader=loader)
i = env.from_string(i).render() # render jinja2
# i = jinja2.Template(i).render() # render jinja2
d = json.loads(i)
if args.alwaysjson:
if pygments:
i = highlight(out, JsonLexer(), formatter())
print(i)
else:
out = yaml.safe_dump(d, indent=args.indent, allow_unicode=True)
if pygments:
out = highlight(out, YamlLexer(), formatter())
print(out)
except json.decoder.JSONDecodeError:
try: # ... to read yaml
d = yaml.safe_load(i) # plain load was deprecated in PyYAML
out = json.dumps(d, indent=args.indent)
if pygments:
out = highlight(out, JsonLexer(), formatter())
print(out)
except yaml.parser.ParserError as exception:
print("input error: invalid json or yaml format", file=sys.stderr)
print(exception, file=sys.stderr)
def load(source, **kwargs):
return yaml.safe_load(source, **kwargs)
import ruamel_yaml
import pytest
from argparse import ArgumentParser
PARSER = ArgumentParser()
PARSER.add_argument("--ignore", default="")
ARGS = PARSER.parse_args()
# paths unlikely to change per package
LICENSE_DIR = "library_licenses"
RECIPE_DIR = Path(os.environ["RECIPE_DIR"])
SRC_DIR = Path(os.environ["SRC_DIR"])
# semi-surpisingly, this is the post-rendered recipe
META = ruamel_yaml.safe_load((RECIPE_DIR / "meta.yaml").read_text("utf-8"))
META_LICENSE_NAMES = [
lf.split(f"{LICENSE_DIR}/")[1]
for lf in META["about"]["license_file"]
if LICENSE_DIR in lf
]
# vendored crates are covered by packaged LICENSE-* file
IGNORE_DEPS = ARGS.ignore.strip().split(",")
LIBRARY_LICENSES = RECIPE_DIR / LICENSE_DIR
LICENSE_FILE_NAMES = sorted([f.name for f in LIBRARY_LICENSES.glob("*")])
# as generated by cargo-license
RAW_DEPENDENCIES = json.loads(
(
