def dump_yaml_by_category(items, root_dir=None):
""" Dumps a set of files: ``$root_dir/$category/$activity.yaml``
"""
assert root_dir
categorized = defaultdict(lambda: defaultdict(lambda: []))
total_facts = 0
for raw_fact in items:
category = raw_fact["category"]
activity = raw_fact["activity"]
fact = OrderedDict()
keys = "since", "until", "description", "tags", "hamster_fact_id"
for key in keys:
if key in raw_fact:
fact[key] = raw_fact[key]
categorized[category][activity].append(fact)
total_facts += 1
for category in categorized:
for activity, facts in categorized[category].iteritems():
category_dir = os.path.join(root_dir, category)
if not os.path.exists(category_dir):
os.makedirs(category_dir)
activity_file = os.path.join(category_dir, activity) + ".yaml"
with open(activity_file, "w") as f:
yaml.dump(facts, f, allow_unicode=True, default_flow_style=False)
return total_facts
def sync(self, args):
""" Synchronize rtc/repository.yaml file and each rtc repository version hash. """
options, argv = self.parse_args(args[:], self._print_alternative_rtcs)
verbose = options.verbose_flag
sys.stdout.write('# Writing repository.yaml for package distribution\n')
sys.stdout.write('## Parsing RTC directory\n')
package = admin.package.get_package_from_path(os.getcwd())
repos = []
for rtc in admin.rtc.get_rtcs_from_package(package, verbose=verbose):
sys.stdout.write('### RTC %s\n' % rtc.rtcprofile.basicInfo.name)
repo = admin.repository.get_repository_from_path(rtc.path, description=rtc.rtcprofile.basicInfo.description)
repos.append(repo)
repo_file = os.path.join(package.get_rtcpath(), 'repository.yaml')
bak_file = repo_file + wasanbon.timestampstr()
if os.path.isfile(bak_file):
os.remove(bak_file)
import shutil, yaml
shutil.copy(repo_file, bak_file)
dic = yaml.load(open(bak_file, 'r'))
if not dic:
dic = {}
for repo in repos:
if getattr(repo, 'url') != None:
url = repo.url.strip()
else:
url = ''
dic[repo.name] = {'repo_name' : repo.name, 'git': url, 'description':repo.description, 'hash':repo.hash}
yaml.dump(dic, open(repo_file, 'w'), encoding='utf8', allow_unicode=True, default_flow_style=False)
pass
def test_missing_files(project_dir, temp_dir, resolve_symlinks, caplog):
args = ['backup', '-t', temp_dir]
if resolve_symlinks:
args.append('--resolve-symlinks')
result = run(args)
assert result.exit_code == 1
assert log_message(logging.ERROR, r'\.FileNotFoundError:', caplog)
extends_content = {'version': '2', 'services': {'none': {'image': 'busybox'}}}
with project_dir.add_file('void.yml').open('tw') as f:
yaml.dump(extends_content, f)
caplog.clear()
result = run(args)
assert result.exit_code == 1
assert log_message(logging.ERROR, r'build path .*/lost either does not exist', caplog)
project_dir.add_folder('lost')
caplog.clear()
result = run(args)
assert result.exit_code == 1
assert log_message(logging.ERROR, r"Couldn't find env file: .*/dangling_link$", caplog)
project_dir.add_file('gone')
caplog.clear()
assert result_okay(run(args))
def dump_yaml_by_year_week(items, root_dir=None):
""" Dumps a set of files: ``$root_dir/$year/$week_number.yaml``
"""
assert root_dir
years = defaultdict(lambda: defaultdict(lambda: []))
total_facts = 0
for raw_fact in items:
year = raw_fact["since"].year
week = raw_fact["since"].isocalendar()[1]
fact = OrderedDict()
keys = "activity", "category", "since", "until", "description", "tags", "hamster_fact_id"
for key in keys:
if key in raw_fact:
fact[key] = raw_fact[key]
years[year][week].append(fact)
total_facts += 1
for year in years:
for month, facts in years[year].iteritems():
year_dir = os.path.join(root_dir, str(year))
if not os.path.exists(year_dir):
os.makedirs(year_dir)
month_file = os.path.join(year_dir, "{0:0>2}".format(month)) + ".yaml"
with open(month_file, "w") as f:
yaml.dump(facts, f, allow_unicode=True, default_flow_style=False)
return total_facts
def get_post_process_yaml(data_dir, workdir):
try:
from bcbiovm.docker.defaults import get_datadir
datadir = get_datadir()
system = os.path.join(datadir, "galaxy", "bcbio_system.yaml") if datadir else None
except ImportError:
system = None
if system is None or not os.path.exists(system):
try:
_, system = load_system_config("bcbio_system.yaml")
except ValueError:
system = None
sample = os.path.join(data_dir, "post_process-sample.yaml")
std = os.path.join(data_dir, "post_process.yaml")
if os.path.exists(std):
return std
elif system and os.path.exists(system):
# create local config pointing to reduced genomes
test_system = os.path.join(workdir, os.path.basename(system))
with open(system) as in_handle:
config = yaml.load(in_handle)
config["galaxy_config"] = os.path.join(data_dir, "universe_wsgi.ini")
with open(test_system, "w") as out_handle:
yaml.dump(config, out_handle)
return test_system
else:
return sample
def _merge_system_configs(host_config, container_config, out_file=None):
"""Create a merged system configuration from external and internal specification.
"""
out = copy.deepcopy(container_config)
for k, v in host_config.iteritems():
if k in set(["galaxy_config"]):
out[k] = v
elif k == "resources":
for pname, resources in v.iteritems():
if not isinstance(resources, dict) and pname not in out[k]:
out[k][pname] = resources
else:
for rname, rval in resources.iteritems():
if rname in set(["cores", "jvm_opts", "memory"]):
if pname not in out[k]:
out[k][pname] = {}
out[k][pname][rname] = rval
# Ensure final file is relocatable by mapping back to reference directory
if "bcbio_system" in out and ("galaxy_config" not in out or not os.path.isabs(out["galaxy_config"])):
out["galaxy_config"] = os.path.normpath(os.path.join(os.path.dirname(out["bcbio_system"]),
os.pardir, "galaxy",
"universe_wsgi.ini"))
if out_file:
with open(out_file, "w") as out_handle:
yaml.dump(out, out_handle, default_flow_style=False, allow_unicode=False)
return out
def main():
root = "/mnt/charts/docs"
chart_url = os.environ.get(
"CHARTS_URL", "https://kubernetes-charts.storage.googleapis.com/")
repo_url = os.environ.get("GIT_REPO")
if repo_url is None:
raise RuntimeError("You must specify a git repo!")
p = urlparse(repo_url)
git_user = p.path.split("/")[-2]
repo_name = p.path.split("/")[-1].split(".")[0]
default_mirror = "https://%s.github.io/%s/" % (git_user.lower(), repo_name)
mirror_url = os.environ.get("MIRROR_URL", default_mirror)
index_file = "index.yaml"
wget(chart_url + index_file, index_file)
with open(index_file) as f:
index = yaml.load(f)
entries = index["entries"]
new = index.copy()
for name, charts in entries.items():
for chart, new_chart in zip(charts, new["entries"][name]):
url = chart["urls"][0]
tar_name = url.split("/")[-1]
target = os.path.join(root, tar_name)
new_chart["urls"][0] = "/".join(
[mirror_url[:-1] if mirror_url.endswith("/") else mirror_url, tar_name])
# datetime format issure
new_chart["created"] = new_chart["created"].strftime('%Y-%m-%dT%H:%M:%S.%f000Z')
if os.path.exists(target):
continue
wget(url, target)
new["generated"] = new["generated"].strftime('%Y-%m-%dT%H:%M:%S.%f000Z')
with open(os.path.join(root, "index.yaml"), "w") as f:
yaml.dump(new, stream=f)
def save(self) -> 'Bundle':
"""Save the bundle to disk."""
if not self._bundle_file:
raise Exception()
with open(self._bundle_file, 'w') as f:
yaml.dump(self, f, **Bundle.YAML_DUMP_ARGS)
return self
def test_disks_flag(self):
# specifying a EBS mount or PD mount is only valid for EC2/Euca/GCE, so
# fail on a cluster deployment.
argv = self.cluster_argv[:] + ["--disks", "ABCDFEG"]
self.assertRaises(BadConfigurationException, ParseArgs, argv, self.function)
# if we get a --disk flag, fail if it's not a dict (after base64, yaml load)
bad_disks_layout = yaml.load("""
public1,
""")
base64ed_bad_disks = base64.b64encode(yaml.dump(bad_disks_layout))
cloud_argv1 = self.cloud_argv[:] + ["--disks", base64ed_bad_disks]
self.assertRaises(BadConfigurationException, ParseArgs, cloud_argv1,
self.function)
# passing in a dict should be fine, and result in us seeing the same value
# for --disks that we passed in.
disks = {'public1' : 'vol-ABCDEFG'}
good_disks_layout = yaml.load("""
public1 : vol-ABCDEFG
""")
base64ed_good_disks = base64.b64encode(yaml.dump(good_disks_layout))
cloud_argv2 = self.cloud_argv[:] + ["--disks", base64ed_good_disks]
actual = ParseArgs(cloud_argv2, self.function).args
self.assertEquals(disks, actual.disks)
def test_unsafe(self):
dummy = Dummy()
with self.assertRaises(yaml.representer.RepresenterError):
yaml.dump_all([dummy])
with self.assertRaises(yaml.representer.RepresenterError):
yaml.dump(dummy, Dumper=yDumper)
# reverse monkey patch and try again
monkey_patch_pyyaml_reverse()
with tempfile.TemporaryFile(suffix='.yaml') as f:
yaml.dump_all([dummy], stream=f)
f.seek(0) # rewind
doc_unsafe = yaml.load(f)
self.assertTrue(type(doc_unsafe) is Dummy)
monkey_patch_pyyaml()
with self.assertRaises(yaml.constructor.ConstructorError):
f.seek(0) # rewind
safe_yaml_load(f)
with self.assertRaises(yaml.constructor.ConstructorError):
f.seek(0) # rewind
yaml.load(f)
def settings(request):
profile = request.user.get_profile()
merchant_settings = MerchantSettings.load_by_merchant(profile)
settings = MerchantSettingsForm(instance=merchant_settings)
account_info = dict()
try:
cb_api = coinbase.get_api_instance(profile)
account_info = coinbase.get_account_info(cb_api)
print yaml.dump(account_info)
except Exception as e:
print "%s: %s" % (e.__class__, e)
cb_api = None
transactions = list()
# if cb_api:
# try:
# transactions = [x for x in cb_api.transactions()]
# if len(transactions) > 7:
# transactions = transactions[:7]
# except Exception as e:
# print "Exception getting transactions: %s %s" % (e.__class__, e)
# for tx in cb_api.transactions():
# print tx
# print dir(tx)
data = {'settings_form': settings,
'coinbase_api': cb_api,
'account_info': account_info,
'transactions': transactions,
}
t = loader.get_template("coinexchange/account/settings.html")
c = CoinExchangeContext(request, data)
return HttpResponse(t.render(c))
def populate_config_from_appliances(appliance_data):
"""populates env.local.yaml with the appliances just obtained
args:
appliance_data: the data of the appliances as taken from sprout
"""
file_name = conf_path.join('env.local.yaml').strpath
if os.path.exists(file_name):
with open(file_name) as f:
y_data = yaml.load(f)
if not y_data:
y_data = {}
else:
y_data = {}
if y_data:
with open(conf_path.join('env.local.backup').strpath, 'w') as f:
yaml.dump(y_data, f, default_flow_style=False)
y_data['appliances'] = []
for app in appliance_data:
app_config = dict(
hostname=app['ip_address'],
ui_protocol="https",
version=str(app['template_version']),
)
y_data['appliances'].append(app_config)
with open(file_name, 'w') as f:
# Use safe dump to avoid !!python/unicode tags
yaml.safe_dump(y_data, f, default_flow_style=False)
def gen_maestro_yaml():
with open("maestro.yaml", "w") as conf:
data = {
"name": cluster_name,
"ships": {},
"services": {
"weibo-emotion-app": {
"image": base_image,
"instances": {}
},
},
"audit": [
{"type": "log", "file": "/tmp/maestro.log"}
]
}
for ship in ships:
data["ships"][ship] = {"ip": ship}
for i in xrange(len(ships)):
for j in xrange(num_instance_per_ship):
container_no = i * num_instance_per_ship + j
service_name = "weibo-emotion-app"
container_name = "%s%s" % (service_name, container_no)
container_port = base_port + container_no
data["services"][service_name]["instances"][container_name] = {
"ship": ships[i],
"ports": {"client": {"external": container_port, "exposed": 8000}},
"lifecycle": {
"running": [{"type": "http", "port": "client"}],
},
}
yaml.dump(data, conf, default_flow_style=False)
def evaluate(definition, args, account_info, force: bool):
# extract Senza* meta information
info = definition.pop("SenzaInfo")
info["StackVersion"] = args.version
template = yaml.dump(definition, default_flow_style=False)
definition = evaluate_template(template, info, [], args, account_info)
definition = yaml.load(definition)
components = definition.pop("SenzaComponents", [])
# merge base template with definition
BASE_TEMPLATE.update(definition)
definition = BASE_TEMPLATE
# evaluate all components
for component in components:
componentname, configuration = named_value(component)
configuration["Name"] = componentname
componenttype = configuration["Type"]
componentfn = get_component(componenttype)
if not componentfn:
raise click.UsageError('Component "{}" does not exist'.format(componenttype))
definition = componentfn(definition, configuration, args, info, force)
# throw executed template to templating engine and provide all information for substitutions
template = yaml.dump(definition, default_flow_style=False)
definition = evaluate_template(template, info, components, args, account_info)
definition = yaml.load(definition)
return definition
def main(run_info_yaml, lane, out_file, genome_build, barcode_type, trim, ascii, analysis, description, clear_description, verbose):
if verbose: print "Verifying that %s exists" % run_info_yaml
assert os.path.exists(run_info_yaml)
if verbose: print "Parsing %s" % run_info_yaml
with open(run_info_yaml) as fh:
run_info = yaml.load(fh)
if verbose: print "Extracting lane info"
if lane == 0:
lane_info = run_info
else:
for info in run_info:
if (int(info.get("lane",0)) == lane):
lane_info = [info]
break
for info in lane_info:
if verbose: print "Processing lane %s" % info["lane"]
_process_info(info,genome_build,barcode_type,trim,ascii,analysis,description,clear_description,verbose)
if out_file is not None:
with open(out_file,'w') as fh:
yaml.dump(run_info, fh, allow_unicode=True, default_flow_style=False)
else:
print yaml.dump(run_info, allow_unicode=True, default_flow_style=False)
def _configure_services(self):
"""Configure all of the services."""
u.log.debug("Running all tests in Apparmor enforce mode.")
nova_config = {'config-flags': 'auto_assign_floating_ip=False',
'enable-live-migration': 'False',
'aa-profile-mode': 'enforce'}
nova_cc_config = {}
if self.git:
amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
reqs_repo = 'git://github.com/openstack/requirements'
neutron_repo = 'git://github.com/openstack/neutron'
nova_repo = 'git://github.com/openstack/nova'
if self._get_openstack_release() == self.trusty_icehouse:
reqs_repo = 'git://github.com/coreycb/requirements'
neutron_repo = 'git://github.com/coreycb/neutron'
nova_repo = 'git://github.com/coreycb/nova'
branch = 'stable/' + self._get_openstack_release_string()
openstack_origin_git = {
'repositories': [
{'name': 'requirements',
'repository': reqs_repo,
'branch': branch},
{'name': 'neutron',
'repository': neutron_repo,
'branch': branch},
{'name': 'nova',
'repository': nova_repo,
'branch': branch},
],
'directory': '/mnt/openstack-git',
'http_proxy': amulet_http_proxy,
'https_proxy': amulet_http_proxy,
}
nova_config['openstack-origin-git'] = \
yaml.dump(openstack_origin_git)
nova_cc_config['openstack-origin-git'] = \
yaml.dump(openstack_origin_git)
if self._get_openstack_release() >= self.xenial_ocata:
nova_cc_config['network-manager'] = 'Neutron'
keystone_config = {
'admin-password': '******',
'admin-token': 'ubuntutesting',
}
pxc_config = {
'max-connections': 1000,
}
configs = {
'nova-compute': nova_config,
'keystone': keystone_config,
'nova-cloud-controller': nova_cc_config,
'percona-cluster': pxc_config,
}
super(NovaBasicDeployment, self)._configure_services(configs)
def main():
arg = sys.argv[1:]
json_file = open(arg[0], 'r')
json_obj = json.load(json_file, encoding="latin-1")
json_obj = byteify(json_obj)
path = os.getcwd()
for base_object in json_obj:
if '.' not in base_object:
new_directory = "%s/inventory/%s" % (path, base_object)
if not os.path.exists(new_directory):
os.makedirs(new_directory)
for group_vars in json_obj['group_vars']:
stream = open("%s/inventory/group_vars/%s" % (path, group_vars), "w")
yaml.dump(json_obj['group_vars'][group_vars], stream, default_flow_style=False)
stream.close()
for host_vars in json_obj['host_vars']:
stream = open("%s/inventory/host_vars/%s" % (path, host_vars), "w")
yaml.dump(json_obj['host_vars'][host_vars], stream, default_flow_style=False)
stream.close()
stream = open("%s/inventory/hosts.ini" % path, "w")
stream.write( json_obj['hosts.ini'] )
stream.close()
def test_yaml_representation_has_all_expected_fields(self):
"""Verify that the YAML representation of reference props is ok."""
prop = properties.ReferenceProperty(
'name', references.Reference(
'5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0', None, None))
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, dict))
self.assertEqual(data['uuid'], '5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0')
self.assertTrue(not 'service' in data)
self.assertTrue(not 'ref' in data)
prop = properties.ReferenceProperty(
'name', references.Reference(
'5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0', 'issues', None))
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, dict))
self.assertEqual(data['uuid'], '5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0')
self.assertEqual(data['service'], 'issues')
self.assertTrue(not 'ref' in data)
prop = properties.ReferenceProperty(
'name', references.Reference(
'5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0',
'issues', 'master'))
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, dict))
self.assertEqual(data['uuid'], '5f2c9a1d-1113-49f1-9d1d-29aaa4a520b0')
self.assertEqual(data['service'], 'issues')
self.assertEqual(data['ref'], 'master')
def test_yaml_representation_has_all_expected_fields(self):
"""Verify that the YAML representation of list properties is ok."""
prop = properties.ListProperty('name', [])
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, list))
self.assertEqual(len(data), 0)
prop = properties.ListProperty('name', [
properties.IntProperty('name', 5),
properties.IntProperty('name', -17),
])
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, list))
self.assertEqual(len(data), 2)
self.assertEqual(data[0], 5)
self.assertEqual(data[1], -17)
prop = properties.ListProperty('name', [
properties.TextProperty('name', 'foo'),
properties.TextProperty('name', 'bar'),
properties.TextProperty('name', 'baz'),
])
string = yaml.dump(prop)
data = yaml.load(string)
self.assertTrue(isinstance(data, list))
self.assertEqual(len(data), 3)
self.assertEqual(data[0], 'foo')
self.assertEqual(data[1], 'bar')
self.assertEqual(data[2], 'baz')
def setup_minion(*roles):
"""Setup a minion server with a set of roles."""
require('environment')
for r in roles:
if r not in VALID_ROLES:
abort('%s is not a valid server role for this project.' % r)
config = {
'master': 'localhost' if env.master == env.host else env.master,
'output': 'mixed',
'grains': {
'environment': env.environment,
'roles': list(roles),
},
'mine_functions': {
'network.interfaces': [],
'network.ip_addrs': []
},
}
_, path = tempfile.mkstemp()
with open(path, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
sudo("mkdir -p /etc/salt")
put(local_path=path, remote_path="/etc/salt/minion", use_sudo=True)
# install salt minion if it's not there already
install_salt(SALT_VERSION, master=False, minion=True, restart=True)
# queries server for its fully qualified domain name to get minion id
key_name = run('python -c "import socket; print socket.getfqdn()"')
execute(accept_key, key_name)
def main():
if len(sys.argv) < 2:
print 'Usage: python rectangle.py FOLDER'
sys.exit(1)
cvStartWindowThread()
directory = os.path.expanduser(sys.argv[1])
filename = raw_input('Save results to what file? ')
config = {}
try:
config = yaml.load(file(filename))
print 'WARNING: Updating file', os.path.abspath(filename)
except IOError:
print 'Creating new file', os.path.abspath(filename)
files = [os.path.join(directory, x) for x in os.listdir(directory)]
#cvNamedWindow('Image', 1)
try:
for i, f in enumerate(files):
img = cvLoadImage(f)
display = Display(os.path.basename(f), img)
display.execute(config)
cvReleaseImage(img)
print '\r%d / %d' % (i+1, len(files)),
sys.stdout.flush()
#cvDestroyWindow('Image')
except:
pass
with open(os.path.expanduser(filename), 'w') as fd:
yaml.dump(config, fd)
def build_tagfile(apt_deps, tags_db, rosdoc_tagfile, current_package, ordered_deps, docspace, ros_distro):
#Get the relevant tags from the database
tags = []
for dep in apt_deps:
if tags_db.has_tags(dep):
#Make sure that we don't pass our own tagfile to ourself
#bad things happen when we do this
for tag in tags_db.get_tags(dep):
if tag['package'] != current_package:
tags.append(tag)
#Add tags built locally in dependency order
for dep in ordered_deps:
#we'll exit the loop when we reach ourself
if dep == current_package:
break
relative_tags_path = "doc/%s/api/%s/tags/%s.tag" % (ros_distro, dep, dep)
if os.path.isfile(os.path.join(docspace, relative_tags_path)):
tags.append({'docs_url': '../../api/%s/html' % dep,
'location': 'file://%s' % os.path.join(docspace, relative_tags_path),
'package': '%s' % dep})
else:
print "DID NOT FIND TAG FILE at: %s" % os.path.join(docspace, relative_tags_path)
with open(rosdoc_tagfile, 'w+') as tags_file:
yaml.dump(tags, tags_file)
def test_print_basic(monkeypatch):
monkeypatch.setattr('boto.cloudformation.connect_to_region', lambda x: MagicMock())
monkeypatch.setattr('boto.iam.connect_to_region', lambda x: MagicMock())
data = {'SenzaInfo': {'StackName': 'test'}, 'SenzaComponents': [{'Configuration': {'Type': 'Senza::Configuration',
'ServerSubnets': {
'eu-west-1': [
'subnet-123']}}},
{'AppServer': {
'Type': 'Senza::TaupageAutoScalingGroup',
'InstanceType': 't2.micro',
'Image': 'AppImage',
'TaupageConfig': {'runtime': 'Docker',
'source': 'foo/bar'}}}]}
runner = CliRunner()
with runner.isolated_filesystem():
with open('myapp.yaml', 'w') as fd:
yaml.dump(data, fd)
result = runner.invoke(cli, ['print', 'myapp.yaml', '--region=myregion', '123', '1.0-SNAPSHOT'],
catch_exceptions=False)
assert 'AWSTemplateFormatVersion' in result.output
assert 'subnet-123' in result.output
def test_console(monkeypatch):
stack = MagicMock(stack_name='test-1')
inst = MagicMock()
inst.tags = {'aws:cloudformation:stack-name': 'test-1'}
ec2 = MagicMock()
ec2.get_only_instances.return_value = [inst]
ec2.get_console_output.return_value.output = b'**MAGIC-CONSOLE-OUTPUT**'
monkeypatch.setattr('boto.ec2.connect_to_region', lambda x: ec2)
monkeypatch.setattr('boto.cloudformation.connect_to_region',
lambda x: MagicMock(list_stacks=lambda stack_status_filters: [stack]))
monkeypatch.setattr('boto.iam.connect_to_region', lambda x: MagicMock())
runner = CliRunner()
data = {'SenzaInfo': {'StackName': 'test'}}
with runner.isolated_filesystem():
with open('myapp.yaml', 'w') as fd:
yaml.dump(data, fd)
result = runner.invoke(cli, ['console', 'myapp.yaml', '--region=myregion', '1'],
catch_exceptions=False)
assert '**MAGIC-CONSOLE-OUTPUT**' in result.output
result = runner.invoke(cli, ['console', 'foobar', '--region=myregion'],
catch_exceptions=False)
assert '' == result.output
result = runner.invoke(cli, ['console', '172.31.1.2', '--region=myregion'],
catch_exceptions=False)
assert '**MAGIC-CONSOLE-OUTPUT**' in result.output
result = runner.invoke(cli, ['console', 'i-123', '--region=myregion'],
catch_exceptions=False)
assert '**MAGIC-CONSOLE-OUTPUT**' in result.output
def test_delete(monkeypatch):
cf = MagicMock()
stack = MagicMock(stack_name='test-1')
cf.list_stacks.return_value = [stack]
monkeypatch.setattr('boto.cloudformation.connect_to_region', lambda x: cf)
monkeypatch.setattr('boto.iam.connect_to_region', lambda x: MagicMock())
runner = CliRunner()
data = {'SenzaInfo': {'StackName': 'test'}}
with runner.isolated_filesystem():
with open('myapp.yaml', 'w') as fd:
yaml.dump(data, fd)
result = runner.invoke(cli, ['delete', 'myapp.yaml', '--region=myregion', '1'],
catch_exceptions=False)
assert 'OK' in result.output
cf.list_stacks.return_value = [stack, stack]
result = runner.invoke(cli, ['delete', 'myapp.yaml', '--region=myregion'],
catch_exceptions=False)
assert 'Please use the "--force" flag if you really want to delete multiple stacks' in result.output
result = runner.invoke(cli, ['delete', 'myapp.yaml', '--region=myregion', '--force'],
catch_exceptions=False)
assert 'OK' in result.output
def __init__(self):
configure(None)
self.logger = getLogger("Permissions")
self.confdir = tmpdir + "/config/"
self.datadir = tmpdir + "/data/"
try:
os.makedirs(self.confdir)
os.makedirs(self.datadir)
self.logger.debug("Config and data dirs created.")
except Exception:
pass
yaml.dump({"editor_warning": False},
open(self.confdir + "settings.yml", "w"))
self.storage = StorageManager(self.confdir, self.datadir)
self.data = self.storage.get_file(self, "data", formats.YAML,
"permissions.yml")
self.handler = permissionsHandler(self, self.data)
super(TestPlugin, self).__init__(
AttrDict(name="test", module="test_permissions"),
AttrDict(name="python"),
)
def save_config(config, path=None):
""" Save given quaycon configuration to YAML file
:param dict config:
configuration to save
:param path:
where to save the configuration. Can be either a path to a file, or
an opened file-like object. If kept to `None`, then configuration is
saved to ~/.quaycon.yml if it already exist or
~/.config/quaycon/config.yml
"""
if path is None:
for _path in DEFAULT_CONFIG_FILES:
if osp.exists(_path):
path = _path
break
if path is None:
path = _path
if isinstance(path, string_types):
LOGGER.info("Saving config in '{}'".format(path))
parent = osp.dirname(path)
if not osp.isdir(parent):
os.makedirs(parent)
with open(path, "w") as ostr:
yaml.dump(config, ostr, encoding="utf-8", default_flow_style=False, Dumper=Dumper)
else:
yaml.dump(config, path, encoding="utf-8", default_flow_style=False, Dumper=Dumper)
def test_print_replace_mustache(monkeypatch):
sg = MagicMock()
sg.name = 'app-master-mind'
sg.id = 'sg-007'
monkeypatch.setattr('boto.cloudformation.connect_to_region', lambda x: MagicMock())
monkeypatch.setattr('boto.ec2.connect_to_region', lambda x: MagicMock(get_all_security_groups=lambda: [sg]))
monkeypatch.setattr('boto.iam.connect_to_region', lambda x: MagicMock())
data = {'SenzaInfo': {'StackName': 'test',
'Parameters': [{'ApplicationId': {'Description': 'Application ID from kio'}}]},
'SenzaComponents': [{'Configuration': {'ServerSubnets': {'eu-west-1': ['subnet-123']},
'Type': 'Senza::Configuration'}},
{'AppServer': {'Image': 'AppImage',
'InstanceType': 't2.micro',
'SecurityGroups': ['app-{{Arguments.ApplicationId}}'],
'IamRoles': ['app-{{Arguments.ApplicationId}}'],
'TaupageConfig': {'runtime': 'Docker',
'source': 'foo/bar'},
'Type': 'Senza::TaupageAutoScalingGroup'}}]
}
runner = CliRunner()
with runner.isolated_filesystem():
with open('myapp.yaml', 'w') as fd:
yaml.dump(data, fd)
result = runner.invoke(cli, ['print', 'myapp.yaml', '--region=myregion', '123', 'master-mind'],
catch_exceptions=False)
assert 'AWSTemplateFormatVersion' in result.output
assert 'subnet-123' in result.output
assert 'app-master-mind' in result.output
assert 'sg-007' in result.output
def dump_yaml_by_year_month_day(items, root_dir=None):
""" Dumps a set of files: ``$root_dir/$year/$month/$day.yaml``
"""
assert root_dir
years = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: [])))
total_facts = 0
for raw_fact in items:
year = raw_fact["since"].year
month = raw_fact["since"].month
day = raw_fact["since"].day
fact = OrderedDict()
keys = "activity", "category", "since", "until", "description", "tags", "hamster_fact_id"
for key in keys:
if key in raw_fact:
fact[key] = raw_fact[key]
years[year][month][day].append(fact)
total_facts += 1
for year in years:
for month in years[year]:
for day, facts in years[year][month].iteritems():
month_dir = os.path.join(root_dir, str(year), "{0:0>2}".format(month))
if not os.path.exists(month_dir):
os.makedirs(month_dir)
day_file = os.path.join(month_dir, "{0:0>2}".format(day)) + ".yaml"
with open(day_file, "w") as f:
yaml.dump(facts, f, allow_unicode=True, default_flow_style=False)
return total_facts
def openstack_client_config(engine):
cloud_config = {
'clouds': {
DEFAULT_CLOUD: {
'auth': {
'auth_url': OPENSTACK_AUTH_URL.format(engine.ip()),
'username': OPENSTACK_USERNAME,
'password': engine.metadata['ovirt-engine-password']
},
'verify': False
}
}
}
os_client_config_file_path = os.path.join(
os.environ.get('SUITE'), OPENSTACK_CLIENT_CONFIG_FILE
)
with open(os_client_config_file_path, 'w') as cloud_config_file:
yaml.dump(cloud_config, cloud_config_file, default_flow_style=False)
original_os_client_config_file = os.environ.get('OS_CLIENT_CONFIG_FILE')
os.environ['OS_CLIENT_CONFIG_FILE'] = os_client_config_file_path
yield DEFAULT_CLOUD
if original_os_client_config_file is not None:
os.environ['OS_CLIENT_CONFIG_FILE'] = original_os_client_config_file
else:
del os.environ['OS_CLIENT_CONFIG_FILE']
def yaml_fwrite(filepath, content, preamble=""):
with open(filepath, "w") as f:
f.write(preamble)
f.write(yaml.dump(content, Dumper=YamlDumper))
def run(self):
with NamedTemporaryFile(delete=True) as playbook_file:
playbook_str = yaml.dump([self.playbook])
playbook_file.write(playbook_str)
playbook_file.flush()
with NamedTemporaryFile(delete=True) as ansible_cfg:
ansible_str = """
[defaults]
host_key_checking = False
stdout_callback = debug
remote_tmp = $HOME/.ansible/tmp
"""
ansible_cfg.write(ansible_str)
ansible_cfg.flush()
with NamedTemporaryFile(delete=True) as hosts_file:
inventory_str = self._build_host_file_contents()
hosts_file.write(inventory_str)
hosts_file.flush()
options = {
'subset': None,
'ask_pass': False,
'listtags': None,
'become_method': self.become_method or 'sudo',
'become_user': self.become_user or 'root',
'sudo': False,
'private_key_file': None,
'syntax': None,
'skip_tags': [],
'diff': False,
'sftp_extra_args': '',
'check': False,
'force_handlers': False,
'remote_user': None,
'become_method': u'sudo',
'vault_password_file': None,
'listtasks': None,
'output_file': None,
'ask_su_pass': False,
'new_vault_password_file': None,
'inventory': u'hosts',
'forks': 100,
'listhosts': None,
'ssh_extra_args': '',
'tags': [u'all'],
'become_ask_pass': False,
'start_at_task': None,
'flush_cache': None,
'step': None,
'become': True,
'su_user': None,
'ask_sudo_pass': False,
'extra_vars': [],
'verbosity': 3,
'su': False,
'ssh_common_args': '',
'connection': 'ssh',
'ask_vault_pass': False,
'timeout': 10,
'module_path': None,
'sudo_user': None,
'scp_extra_args': ''
}
os.environ['ANSIBLE_CONFIG'] = ansible_cfg.name
from ansible_utils import Runner
runner = Runner(
playbook=playbook_file.name,
hosts=hosts_file.name,
options=options
)
stats = runner.run()
"kubernetesStorageVersion": "v1",
"openShiftStoragePrefix": "openshift.io",
"openShiftStorageVersion": "v1"
}
# k8s service account authentication
config['serviceAccountConfig']['privateKeyFile'] = "/etc/kubernetes/ssl/key.pem"
config['serviceAccountConfig']['publicKeyFiles'] = ["/etc/kubernetes/ssl/cert.pem"]
# configure kubelet access to allow tailing sti-build log files
config['kubeletClientInfo'] = {
"ca": "/etc/kubernetes/ssl/ca.pem",
"certFile": "/etc/kubernetes/ssl/cert.pem",
"keyFile": "/etc/kubernetes/ssl/key.pem",
"port": 10250
}
# user authentication
config['oauthConfig']['identityProviders'][0] = {
"name": "rancher",
"challenge": True,
"login": True,
"provider": {
"apiVersion": "v1",
"kind": "HTPasswdPasswordIdentityProvider",
"file": "/users.htpasswd"
}
}
yaml.dump(config, file(path, 'w'), default_flow_style=False)
def CallBackFunc(event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
print('left button is clicked - position (', x, ',', y, ')')
list_points.append([x, y])
elif event == cv2.EVENT_RBUTTONDOWN:
print('right button is clicked - position (', x, ',', y, ')')
list_points.append([x, y])
cv2.setMouseCallback('Mousecallback', CallBackFunc)
if __name__ == '__main__':
while (True):
cv2.imshow('Mousecallback', img)
if len(list_points) == 4:
config_data = dict(image_parameters=dict(p2=list_points[3],
p1=list_points[2],
p4=list_points[0],
p3=list_points[1],
width_og=width,
height_og=height,
img_path=img_path,
size_frame=size_frame))
with open('../conf/config_birdview.yml', 'w') as outfile:
yaml.dump(config_data, outfile, default_flow_style=False)
break
if cv2.waitKey(20) == 27:
break
cv2.destroyAllWindows()
#!/usr/bin/env python
import yaml
docker_compose_path = "/etc/contrail/config/docker-compose.yaml"
with open(docker_compose_path) as f:
docker_compose = yaml.load(f)
schema = docker_compose["services"]["devicemgr"]
schema["image"] = "pawelzny/contrail-controller-config-devicemgr:R6.0-1"
environment = schema.setdefault("environment", [])
notification_driver = "NOTIFICATION_DRIVER=etcd"
db_driver = "DB_DRIVER=etcd"
if notification_driver not in environment:
environment.append(notification_driver)
if db_driver not in environment:
environment.append(db_driver)
with open(docker_compose_path, "w") as f:
yaml.dump(docker_compose, f)
# Save results
data = {
"rotation_matrix": np.asarray(R).tolist(),
"translation_vector": np.asarray(T).tolist(),
"essential_matrix": np.asarray(E).tolist(),
"fundamental_matrix": np.asarray(F).tolist(),
"R1": np.asarray(R1).tolist(),
"R2": np.asarray(R2).tolist(),
"P1": np.asarray(P1).tolist(),
"P2": np.asarray(P2).tolist(),
"Q": np.asarray(Q).tolist(),
"roi_1": np.asarray(roi1).tolist(),
"roi_2": np.asarray(roi2).tolist()
}
yaml.dump(data, open(SAVE_PATH, 'w'))
np.savez_compressed(SAVE_PATH_MAPS,
ax=map_a_x,
ay=map_a_y,
bx=map_b_x,
by=map_b_y)
# Testing
test_a = cv2.imread(IMG_PATH_A + fnames[0])
test_b = cv2.imread(IMG_PATH_B + fnames[0])
cv2.imshow("Original A", test_a)
cv2.imshow("Original B", test_b)
remapped_a = cv2.remap(test_a, map_a_x, map_a_y, cv2.INTER_LINEAR)
remapped_b = cv2.remap(test_b, map_b_x, map_b_y, cv2.INTER_LINEAR)
cv2.imshow("Remapped A", remapped_a)
def save_yaml_config(filepath, config):
with open(filepath, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
def LogesticRegression(dataFile,outputfolder,C,Penalty,parameters): inputData = yaml.load(open(dataFile)) trainingSet = inputData['training'] testingSet = inputData['testing'] inputFile = inputData['inputFile'] label = inputData['label'] resultset = [] modelset = [] importanceset = [] if not os.path.exists(outputfolder): try: os.makedirs(outputfolder) except OSError as exc: if exc.errno != errno.EEXIST: raise exc pass modelsfolder = outputfolder + "/models/" if not os.path.exists(modelsfolder): try: os.makedirs(modelsfolder) except OSError as exc: if exc.errno != errno.EEXIST: raise exc pass importancefolder = outputfolder + "/FeatureImportance/" if not os.path.exists(importancefolder): try: os.makedirs(importancefolder) except OSError as exc: if exc.errno != errno.EEXIST: raise exc pass for i in range(len(trainingSet)): train_df = pd.read_csv(trainingSet[i]) train_labels = train_df[label] train_features = train_df.drop(label,axis=1) test_df = pd.read_csv(testingSet[i]) test_predictions = pd.DataFrame(test_df[label]) test_features = test_df.drop(label,axis=1) lr = LogisticRegression(penalty=Penalty,C=C) lr.fit(train_features,train_labels) modelFile = modelsfolder + "LogisticRegressionModel" + str(i+1) + ".pkl" with open(modelFile, 'wb') as fd: pickle.dump(lr, fd) modelset.append(modelFile) fd.close() importanceFile = calculateFeatureImportance(train_features,lr,importancefolder,i) importanceset.append(importanceFile) test_predictions['predictions'] = lr.predict(test_features) resultFile = outputfolder+'/result'+str(i+1)+'.csv' test_predictions.to_csv(resultFile,index=False) resultset.append(resultFile) resultDict = dict() resultDict['results'] = resultset resultDict['models'] = modelset resultDict['featureimportance'] = importanceset resultDict['label'] = label if not parameters: parameters['parameter']='default' resultDict['algo_params'] = parameters resultDict['split_params'] = inputData['split_params'] if 'feature_selection_parameters' in inputData: resultDict['feature_selection_parameters'] = inputData['feature_selection_parameters'] resultDict['feature_selection_algorithm'] = inputData['feature_selection_algorithm'] if 'feature_extraction_parameters' in inputData: resultDict['feature_extraction_parameters'] = inputData['feature_extraction_parameters'] resultDict['feature_extraction_algorithm'] = inputData['feature_extraction_algorithm'] if 'preprocessing_params' in inputData: resultDict['preprocessing_params'] = inputData['preprocessing_params'] resultDict['inputFile'] = inputFile resultDict['algorithm'] = "LogisticRegression" yaml.dump(resultDict,open(outputfolder+'/results.yaml','w'))
def train(hyp, opt, device, tb_writer=None, wandb=None):
logger.info(f'Hyperparameters {hyp}')
save_dir, epochs, batch_size, total_batch_size, weights, rank = \
Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank
# Directories
wdir = save_dir / 'weights'
wdir.mkdir(parents=True, exist_ok=True) # make dir
last = wdir / 'last.pt'
best = wdir / 'best.pt'
results_file = save_dir / 'results.txt'
# Save run settings
with open(save_dir / 'hyp.yaml', 'w') as f:
yaml.dump(hyp, f, sort_keys=False)
with open(save_dir / 'opt.yaml', 'w') as f:
yaml.dump(vars(opt), f, sort_keys=False)
# Configure
plots = not opt.evolve # create plots
cuda = device.type != 'cpu'
init_seeds(2 + rank)
with open(opt.data) as f:
data_dict = yaml.load(f, Loader=yaml.FullLoader) # data dict
with torch_distributed_zero_first(rank):
check_dataset(data_dict) # check
train_path = data_dict['train']
test_path = data_dict['val']
nc, names = (1, ['item']) if opt.single_cls else (int(data_dict['nc']), data_dict['names']) # number classes, names
assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check
# Model
pretrained = weights.endswith('.pt')
if pretrained:
with torch_distributed_zero_first(rank):
attempt_download(weights) # download if not found locally
ckpt = torch.load(weights, map_location=device) # load checkpoint
if hyp.get('anchors'):
ckpt['model'].yaml['anchors'] = round(hyp['anchors']) # force autoanchor
model = Model(opt.cfg or ckpt['model'].yaml, ch=3, nc=nc).to(device) # create
exclude = ['anchor'] if opt.cfg or hyp.get('anchors') else [] # exclude keys
state_dict = ckpt['model'].float().state_dict() # to FP32
state_dict = intersect_dicts(state_dict, model.state_dict(), exclude=exclude) # intersect
model.load_state_dict(state_dict, strict=False) # load
logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report
else:
model = Model(opt.cfg, ch=3, nc=nc).to(device) # create
# Freeze
freeze = [] # parameter names to freeze (full or partial)
for k, v in model.named_parameters():
v.requires_grad = True # train all layers
if any(x in k for x in freeze):
print('freezing %s' % k)
v.requires_grad = False
# Optimizer
nbs = 64 # nominal batch size
accumulate = max(round(nbs / total_batch_size), 1) # accumulate loss before optimizing
hyp['weight_decay'] *= total_batch_size * accumulate / nbs # scale weight_decay
pg0, pg1, pg2 = [], [], [] # optimizer parameter groups
for k, v in model.named_modules():
if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter):
pg2.append(v.bias) # biases
if isinstance(v, nn.BatchNorm2d):
pg0.append(v.weight) # no decay
elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter):
pg1.append(v.weight) # apply decay
if opt.adam:
optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum
else:
optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True)
optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay
optimizer.add_param_group({'params': pg2}) # add pg2 (biases)
logger.info('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0)))
del pg0, pg1, pg2
# Scheduler https://arxiv.org/pdf/1812.01187.pdf
# https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#OneCycleLR
lf = lambda x: ((1 + math.cos(x * math.pi / epochs)) / 2) * (1 - hyp['lrf']) + hyp['lrf'] # cosine
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
# plot_lr_scheduler(optimizer, scheduler, epochs)
# Logging
if wandb and wandb.run is None:
opt.hyp = hyp # add hyperparameters
wandb_run = wandb.init(config=opt, resume="allow",
project='YOLOv3' if opt.project == 'runs/train' else Path(opt.project).stem,
name=save_dir.stem,
id=ckpt.get('wandb_id') if 'ckpt' in locals() else None)
loggers = {'wandb': wandb} # loggers dict
# Resume
start_epoch, best_fitness = 0, 0.0
if pretrained:
# Optimizer
if ckpt['optimizer'] is not None:
optimizer.load_state_dict(ckpt['optimizer'])
best_fitness = ckpt['best_fitness']
# Results
if ckpt.get('training_results') is not None:
with open(results_file, 'w') as file:
file.write(ckpt['training_results']) # write results.txt
# Epochs
start_epoch = ckpt['epoch'] + 1
if opt.resume:
assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (weights, epochs)
if epochs < start_epoch:
logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' %
(weights, ckpt['epoch'], epochs))
epochs += ckpt['epoch'] # finetune additional epochs
del ckpt, state_dict
# Image sizes
gs = int(max(model.stride)) # grid size (max stride)
imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples
# DP mode
if cuda and rank == -1 and torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
# SyncBatchNorm
if opt.sync_bn and cuda and rank != -1:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device)
logger.info('Using SyncBatchNorm()')
# EMA
ema = ModelEMA(model) if rank in [-1, 0] else None
# DDP mode
if cuda and rank != -1:
model = DDP(model, device_ids=[opt.local_rank], output_device=opt.local_rank)
# Trainloader
dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt,
hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect, rank=rank,
world_size=opt.world_size, workers=opt.workers,
image_weights=opt.image_weights)
mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class
nb = len(dataloader) # number of batches
assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1)
# Process 0
if rank in [-1, 0]:
ema.updates = start_epoch * nb // accumulate # set EMA updates
testloader = create_dataloader(test_path, imgsz_test, total_batch_size, gs, opt, # testloader
hyp=hyp, cache=opt.cache_images and not opt.notest, rect=True,
rank=-1, world_size=opt.world_size, workers=opt.workers, pad=0.5)[0]
if not opt.resume:
labels = np.concatenate(dataset.labels, 0)
c = torch.tensor(labels[:, 0]) # classes
# cf = torch.bincount(c.long(), minlength=nc) + 1. # frequency
# model._initialize_biases(cf.to(device))
if plots:
Thread(target=plot_labels, args=(labels, save_dir, loggers), daemon=True).start()
if tb_writer:
tb_writer.add_histogram('classes', c, 0)
# Anchors
if not opt.noautoanchor:
check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz)
# Model parameters
hyp['cls'] *= nc / 80. # scale coco-tuned hyp['cls'] to current dataset
model.nc = nc # attach number of classes to model
model.hyp = hyp # attach hyperparameters to model
model.gr = 1.0 # iou loss ratio (obj_loss = 1.0 or iou)
model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) # attach class weights
model.names = names
# Start training
t0 = time.time()
nw = max(round(hyp['warmup_epochs'] * nb), 1000) # number of warmup iterations, max(3 epochs, 1k iterations)
# nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training
maps = np.zeros(nc) # mAP per class
results = (0, 0, 0, 0, 0, 0, 0) # P, R, [email protected], [email protected], val_loss(box, obj, cls)
scheduler.last_epoch = start_epoch - 1 # do not move
scaler = amp.GradScaler(enabled=cuda)
logger.info('Image sizes %g train, %g test\n'
'Using %g dataloader workers\nLogging results to %s\n'
'Starting training for %g epochs...' % (imgsz, imgsz_test, dataloader.num_workers, save_dir, epochs))
for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------
model.train()
# Update image weights (optional)
if opt.image_weights:
# Generate indices
if rank in [-1, 0]:
cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 # class weights
iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights
dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx
# Broadcast if DDP
if rank != -1:
indices = (torch.tensor(dataset.indices) if rank == 0 else torch.zeros(dataset.n)).int()
dist.broadcast(indices, 0)
if rank != 0:
dataset.indices = indices.cpu().numpy()
# Update mosaic border
# b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs)
# dataset.mosaic_border = [b - imgsz, -b] # height, width borders
mloss = torch.zeros(4, device=device) # mean losses
if rank != -1:
dataloader.sampler.set_epoch(epoch)
pbar = enumerate(dataloader)
logger.info(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'total', 'targets', 'img_size'))
if rank in [-1, 0]:
pbar = tqdm(pbar, total=nb) # progress bar
optimizer.zero_grad()
for i, (imgs, targets, paths, _) in pbar: # batch -------------------------------------------------------------
ni = i + nb * epoch # number integrated batches (since train start)
imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0
# Warmup
if ni <= nw:
xi = [0, nw] # x interp
# model.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou)
accumulate = max(1, np.interp(ni, xi, [1, nbs / total_batch_size]).round())
for j, x in enumerate(optimizer.param_groups):
# bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0
x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
if 'momentum' in x:
x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']])
# Multi-scale
if opt.multi_scale:
sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size
sf = sz / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple)
imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False)
# Forward
with amp.autocast(enabled=cuda):
pred = model(imgs) # forward
loss, loss_items = compute_loss(pred, targets.to(device), model) # loss scaled by batch_size
if rank != -1:
loss *= opt.world_size # gradient averaged between devices in DDP mode
# Backward
scaler.scale(loss).backward()
# Optimize
if ni % accumulate == 0:
scaler.step(optimizer) # optimizer.step
scaler.update()
optimizer.zero_grad()
if ema:
ema.update(model)
# Print
if rank in [-1, 0]:
mloss = (mloss * i + loss_items) / (i + 1) # update mean losses
mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB)
s = ('%10s' * 2 + '%10.4g' * 6) % (
'%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1])
pbar.set_description(s)
# Plot
if plots and ni < 3:
f = save_dir / f'train_batch{ni}.jpg' # filename
Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start()
# if tb_writer:
# tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch)
# tb_writer.add_graph(model, imgs) # add model to tensorboard
elif plots and ni == 3 and wandb:
wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg')]})
# end batch ------------------------------------------------------------------------------------------------
# end epoch ----------------------------------------------------------------------------------------------------
# Scheduler
lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard
scheduler.step()
# DDP process 0 or single-GPU
if rank in [-1, 0]:
# mAP
if ema:
ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride'])
final_epoch = epoch + 1 == epochs
if not opt.notest or final_epoch: # Calculate mAP
results, maps, times = test.test(opt.data,
batch_size=total_batch_size,
imgsz=imgsz_test,
model=ema.ema,
single_cls=opt.single_cls,
dataloader=testloader,
save_dir=save_dir,
plots=plots and final_epoch,
log_imgs=opt.log_imgs if wandb else 0)
# Write
with open(results_file, 'a') as f:
f.write(s + '%10.4g' * 7 % results + '\n') # P, R, [email protected], [email protected], val_loss(box, obj, cls)
if len(opt.name) and opt.bucket:
os.system('gsutil cp %s gs://%s/results/results%s.txt' % (results_file, opt.bucket, opt.name))
# Log
tags = ['train/box_loss', 'train/obj_loss', 'train/cls_loss', # train loss
'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95',
'val/box_loss', 'val/obj_loss', 'val/cls_loss', # val loss
'x/lr0', 'x/lr1', 'x/lr2'] # params
for x, tag in zip(list(mloss[:-1]) + list(results) + lr, tags):
if tb_writer:
tb_writer.add_scalar(tag, x, epoch) # tensorboard
if wandb:
wandb.log({tag: x}) # W&B
# Update best mAP
fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, [email protected], [email protected]]
if fi > best_fitness:
best_fitness = fi
# Save model
save = (not opt.nosave) or (final_epoch and not opt.evolve)
if save:
with open(results_file, 'r') as f: # create checkpoint
ckpt = {'epoch': epoch,
'best_fitness': best_fitness,
'training_results': f.read(),
'model': ema.ema,
'optimizer': None if final_epoch else optimizer.state_dict(),
'wandb_id': wandb_run.id if wandb else None}
# Save last, best and delete
torch.save(ckpt, last)
if best_fitness == fi:
torch.save(ckpt, best)
del ckpt
# end epoch ----------------------------------------------------------------------------------------------------
# end training
if rank in [-1, 0]:
# Strip optimizers
for f in [last, best]:
if f.exists(): # is *.pt
strip_optimizer(f) # strip optimizer
os.system('gsutil cp %s gs://%s/weights' % (f, opt.bucket)) if opt.bucket else None # upload
# Plots
if plots:
plot_results(save_dir=save_dir) # save as results.png
if wandb:
files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png']
wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files
if (save_dir / f).exists()]})
logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600))
# Test best.pt
if opt.data.endswith('coco.yaml') and nc == 80: # if COCO
results, _, _ = test.test(opt.data,
batch_size=total_batch_size,
imgsz=imgsz_test,
model=attempt_load(best if best.exists() else last, device).half(),
single_cls=opt.single_cls,
dataloader=testloader,
save_dir=save_dir,
save_json=True, # use pycocotools
plots=False)
else:
dist.destroy_process_group()
wandb.run.finish() if wandb and wandb.run else None
torch.cuda.empty_cache()
return results
def main():
args = parse_args()
root = Path(args.save_path)
load_root = Path(args.load_path) if args.load_path else None
root.mkdir(parents=True, exist_ok=True)
####################################
# Dump arguments and create logger #
####################################
with open(root / "args.yml", "w") as f:
yaml.dump(args, f)
writer = SummaryWriter(str(root))
#######################
# Load PyTorch Models #
#######################
netG = Generator(args.n_mel_channels, args.ngf,
args.n_residual_layers).cuda()
netD = Discriminator(args.num_D, args.ndf, args.n_layers_D,
args.downsamp_factor).cuda()
fft = Audio2Mel(n_mel_channels=args.n_mel_channels).cuda()
print(netG)
print(netD)
#####################
# Create optimizers #
#####################
optG = torch.optim.Adam(netG.parameters(), lr=1e-4, betas=(0.5, 0.9))
optD = torch.optim.Adam(netD.parameters(), lr=1e-4, betas=(0.5, 0.9))
if load_root and load_root.exists():
netG.load_state_dict(torch.load(load_root / "netG.pt"))
optG.load_state_dict(torch.load(load_root / "optG.pt"))
netD.load_state_dict(torch.load(load_root / "netD.pt"))
optD.load_state_dict(torch.load(load_root / "optD.pt"))
#######################
# Create data loaders #
#######################
train_set = AudioDataset(Path(args.data_path) / "train_files.txt",
args.seq_len,
sampling_rate=22050)
test_set = AudioDataset(
Path(args.data_path) / "test_files.txt",
22050 * 4,
sampling_rate=22050,
augment=False,
)
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
num_workers=4)
test_loader = DataLoader(test_set, batch_size=1)
##########################
# Dumping original audio #
##########################
test_voc = []
test_audio = []
for i, x_t in enumerate(test_loader):
x_t = x_t.cuda()
s_t = fft(x_t).detach()
test_voc.append(s_t.cuda())
test_audio.append(x_t)
audio = x_t.squeeze().cpu()
save_sample(root / ("original_%d.wav" % i), 22050, audio)
writer.add_audio("original/sample_%d.wav" % i,
audio,
0,
sample_rate=22050)
if i == args.n_test_samples - 1:
break
costs = []
start = time.time()
# enable cudnn autotuner to speed up training
torch.backends.cudnn.benchmark = True
best_mel_reconst = 1000000
steps = 0
for epoch in range(1, args.epochs + 1):
for iterno, x_t in enumerate(train_loader):
x_t = x_t.cuda()
s_t = fft(x_t).detach()
x_pred_t = netG(s_t.cuda())
with torch.no_grad():
s_pred_t = fft(x_pred_t.detach())
s_error = F.l1_loss(s_t, s_pred_t).item()
#######################
# Train Discriminator #
#######################
D_fake_det = netD(x_pred_t.cuda().detach())
D_real = netD(x_t.cuda())
loss_D = 0
for scale in D_fake_det:
loss_D += F.relu(1 + scale[-1]).mean()
for scale in D_real:
loss_D += F.relu(1 - scale[-1]).mean()
netD.zero_grad()
loss_D.backward()
optD.step()
###################
# Train Generator #
###################
D_fake = netD(x_pred_t.cuda())
loss_G = 0
for scale in D_fake:
loss_G += -scale[-1].mean()
loss_feat = 0
feat_weights = 4.0 / (args.n_layers_D + 1)
D_weights = 1.0 / args.num_D
wt = D_weights * feat_weights
for i in range(args.num_D):
for j in range(len(D_fake[i]) - 1):
loss_feat += wt * F.l1_loss(D_fake[i][j],
D_real[i][j].detach())
netG.zero_grad()
(loss_G + args.lambda_feat * loss_feat).backward()
optG.step()
######################
# Update tensorboard #
######################
costs.append(
[loss_D.item(),
loss_G.item(),
loss_feat.item(), s_error])
writer.add_scalar("loss/discriminator", costs[-1][0], steps)
writer.add_scalar("loss/generator", costs[-1][1], steps)
writer.add_scalar("loss/feature_matching", costs[-1][2], steps)
writer.add_scalar("loss/mel_reconstruction", costs[-1][3], steps)
steps += 1
if steps % args.save_interval == 0:
st = time.time()
with torch.no_grad():
for i, (voc, _) in enumerate(zip(test_voc, test_audio)):
pred_audio = netG(voc)
pred_audio = pred_audio.squeeze().cpu()
save_sample(root / ("generated_%d.wav" % i), 22050,
pred_audio)
writer.add_audio(
"generated/sample_%d.wav" % i,
pred_audio,
epoch,
sample_rate=22050,
)
torch.save(netG.state_dict(), root / "netG.pt")
torch.save(optG.state_dict(), root / "optG.pt")
torch.save(netD.state_dict(), root / "netD.pt")
torch.save(optD.state_dict(), root / "optD.pt")
if np.asarray(costs).mean(0)[-1] < best_mel_reconst:
best_mel_reconst = np.asarray(costs).mean(0)[-1]
torch.save(netD.state_dict(), root / "best_netD.pt")
torch.save(netG.state_dict(), root / "best_netG.pt")
print("Took %5.4fs to generate samples" % (time.time() - st))
print("-" * 100)
if steps % args.log_interval == 0:
print("Epoch {} | Iters {} / {} | ms/batch {:5.2f} | loss {}".
format(
epoch,
iterno,
len(train_loader),
1000 * (time.time() - start) / args.log_interval,
np.asarray(costs).mean(0),
))
costs = []
start = time.time()
1,
x,
y,
len(pixels),
batch_size=7,
pixels_per_grid=49))
pixelids = dict()
for chip_idx, chip in enumerate(chip_ids):
# Bool value is argument to right_side_up
chip_pixels = list(range(chip_idx * 49, chip_idx * 49 + 49))
pixelids[chip] = (True, 'plain', chip_pixels)
chips = []
for chipid, (right_side_up, shape, ids) in pixelids.items():
assignment = pg.grid_7x7_assignments_0_64_v2_2_1
channels = pg.assign_pixels(ids, assignment, right_side_up, range(64))
chips.append([chipid, channels])
print('chips', len(chips))
with open('layout-2.4.0.yaml', 'w') as f:
yaml.dump(
{
'pixels': pixels,
'chips': chips,
'x': -width / 2,
'y': -height / 2,
'width': width,
'height': height
}, f)
with open(filename) as file:
variables = yaml.load(file, Loader=yaml.FullLoader)
variables["host"] = credentials.ip_address
variables["username"] = credentials.username
variables["password"] = credentials.password
variables["tenant"] = temp[1].split('-')[1]
variables["ap"] = temp[2].split('-')[1]
variables["epg"] = temp[3].split('-')[1]
variables["domain"] = domain
filename = os.path.join(dirname,
'ansible/vars/vars_domain_binding.yaml')
with open(filename, "w") as file:
yaml.dump(variables, file)
filename = os.path.join(dirname, 'ansible/bind_physd_epg.yaml')
subprocess.run(['ansible-playbook', filename])
p = subprocess.Popen(['tail', '/var/log/syslog'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.communicate()
print("Physical Domain has been added to EPG!")
response = input("Press Y to add this epg to a VPC:")
if response.lower() == "y":
print("Y pressed!")
def generate_user_data(taupage_config, region):
"""
Generates the CloudFormation "UserData" field.
It looks for AWS functions such as Fn:: and Ref and generates the appropriate UserData json field,
It leaves nodes representing AWS functions or refs unmodified and converts into text everything else.
Example::
environment:
S3_BUCKET: {"Ref": "ExhibitorBucket"}
S3_PREFIX: exhibitor
transforms into::
{"Fn::Join": ["", "environment:\n S3_BUCKET: ", {"Ref": "ExhibitorBucket"}, "\n S3_PREFIX: exhibitor"]}
:param taupage_config:
:return:
"""
def is_aws_fn(name):
try:
return name == "Ref" or (isinstance(name, str)
and name.startswith("Fn::"))
except:
return False
def transform(node):
"""Transform AWS functions and refs into an string representation for later split and substitution"""
if isinstance(node, dict):
num_keys = len(node)
if 'Stack' in node and 'Output' in node:
return resolve_referenced_resource(node, region)
if num_keys > 0:
key = next(iter(node.keys()))
if num_keys == 1 and is_aws_fn(key):
return "".join(["{{ ", json.dumps(node), " }}"])
else:
return {
key: transform(value)
for key, value in node.items()
}
else:
return node
elif isinstance(node, list):
return [transform(subnode) for subnode in node]
else:
return node
def split(text):
"""Splits yaml text into text and AWS functions/refs"""
parts = []
last_pos = 0
for m in _AWS_FN_RE.finditer(text):
parts += [text[last_pos:m.start(1)], json.loads(m.group(2))]
last_pos = m.end(1)
parts += [text[last_pos:]]
return parts
yaml_text = yaml.dump(transform(taupage_config),
width=sys.maxsize,
default_flow_style=False)
parts = split("#taupage-ami-config\n" + yaml_text)
if len(parts) == 1:
return parts[0]
else:
return {"Fn::Join": ["", parts]}
def render(self):
print(yaml.dump(self.data))
YamlVariable.last = None
steps += dask_tests()
steps += dagit_tests()
steps += lakehouse_tests()
steps += pipenv_smoke_tests()
steps += python_modules_tox_tests("dagster")
steps += python_modules_tox_tests("dagster-graphql")
steps += python_modules_tox_tests("dagstermill")
steps += library_tests()
steps += releasability_tests()
if DO_COVERAGE:
steps += [wait_step(), coverage_step()]
print(
yaml.dump(
{
"env": {
"CI_NAME": "buildkite",
"CI_BUILD_NUMBER": "$BUILDKITE_BUILD_NUMBER",
"CI_BUILD_URL": "$BUILDKITE_BUILD_URL",
"CI_BRANCH": "$BUILDKITE_BRANCH",
"CI_PULL_REQUEST": "$BUILDKITE_PULL_REQUEST",
},
"steps": steps,
},
default_flow_style=False,
))
def show_config(self, args):
"""Print active configuration values to console for confirmation"""
stream = yaml.dump(conf)
print(stream.replace('\n-', '\n -'))
def save_yaml_config(filepath, config):
"""Save yaml config file at the given path."""
with open(filepath, 'w') as f:
yaml.dump(config, f, default_flow_style=False)
def dump_key_bindings(obj):
p = key_bindings_path()
with open(p, 'w') as wfile:
yaml.dump(obj, wfile)
async def upgrade(self, job, release_name, options):
"""
Upgrade `release_name` chart release.
`upgrade_options.item_version` specifies to which item version chart release should be upgraded to.
System will update container images being used by `release_name` chart release if
`upgrade_options.update_container_images` is set.
During upgrade, `upgrade_options.values` can be specified to apply configuration changes for configuration
changes for the chart release in question.
For upgrade, system will automatically take a snapshot of `ix_volumes` in question which can be used to
rollback later on.
"""
await self.middleware.call('kubernetes.validate_k8s_setup')
release = await self.middleware.call('chart.release.get_instance',
release_name)
catalog = await self.middleware.call(
'catalog.query',
[['id', '=', release['catalog']]],
{'extra': {
'item_details': True
}},
)
if not catalog:
raise CallError(f'Unable to locate {release["catalog"]!r} catalog',
errno=errno.ENOENT)
else:
catalog = catalog[0]
current_chart = release['chart_metadata']
chart = current_chart['name']
if release['catalog_train'] not in catalog['trains']:
raise CallError(
f'Unable to locate {release["catalog_train"]!r} catalog train in {release["catalog"]!r}',
errno=errno.ENOENT,
)
if chart not in catalog['trains'][release['catalog_train']]:
raise CallError(
f'Unable to locate {chart!r} catalog item in {release["catalog"]!r} '
f'catalog\'s {release["catalog_train"]!r} train.',
errno=errno.ENOENT)
new_version = options['item_version']
if new_version == 'latest':
new_version = await self.middleware.call(
'chart.release.get_latest_version_from_item_versions',
catalog['trains'][release['catalog_train']][chart]['versions'])
if new_version not in catalog['trains'][
release['catalog_train']][chart]['versions']:
raise CallError(
f'Unable to locate specified {new_version!r} item version.')
verrors = ValidationErrors()
if parse_version(new_version) <= parse_version(
current_chart['version']):
verrors.add(
'upgrade_options.item_version',
f'Upgrade version must be greater than {current_chart["version"]!r} current version.'
)
verrors.check()
catalog_item = catalog['trains'][
release['catalog_train']][chart]['versions'][new_version]
await self.middleware.call('catalog.version_supported_error_check',
catalog_item)
config = await self.middleware.call('chart.release.upgrade_values',
release, catalog_item['location'])
# We will be performing validation for values specified. Why we want to allow user to specify values here
# is because the upgraded catalog item version might have different schema which potentially means that
# upgrade won't work or even if new k8s are resources are created/deployed, they won't necessarily function
# as they should because of changed params or expecting new params
# One tricky bit which we need to account for first is removing any key from current configured values
# which the upgraded release will potentially not support. So we can safely remove those as otherwise
# validation will fail as new schema does not expect those keys.
config = clean_values_for_upgrade(config,
catalog_item['schema']['questions'])
config.update(options['values'])
config, context = await self.middleware.call(
'chart.release.normalise_and_validate_values',
catalog_item,
config,
False,
release['dataset'],
)
job.set_progress(25, 'Initial validation complete')
# We have validated configuration now
chart_path = os.path.join(release['path'], 'charts', new_version)
await self.middleware.run_in_thread(shutil.rmtree,
chart_path,
ignore_errors=True)
await self.middleware.run_in_thread(shutil.copytree,
catalog_item['location'],
chart_path)
# If a snapshot of the volumes already exist with the same name in case of a failed upgrade, we will remove
# it as we want the current point in time being reflected in the snapshot
volumes_ds = os.path.join(release['dataset'], 'volumes/ix_volumes')
snap_name = f'{volumes_ds}@{release["version"]}'
if await self.middleware.call('zfs.snapshot.query',
[['id', '=', snap_name]]):
await self.middleware.call('zfs.snapshot.delete', snap_name)
await self.middleware.call('zfs.snapshot.create', {
'dataset': volumes_ds,
'name': release['version'],
'recursive': True
})
job.set_progress(40, 'Created snapshot for upgrade')
await self.middleware.call('chart.release.perform_actions', context)
# Let's update context options to reflect that an upgrade is taking place and from which version to which
# version it's happening.
# Helm considers simple config change as an upgrade as well, and we have no way of determining the old/new
# chart versions during helm upgrade in the helm template, hence the requirement for a context object.
config[CONTEXT_KEY_NAME].update({
'operation': 'UPGRADE',
'isUpgrade': True,
'upgradeMetadata': {
'oldChartVersion': current_chart['version'],
'newChartVersion': new_version,
'preUpgradeRevision': release['version'],
}
})
job.set_progress(50, 'Upgrading chart release')
with tempfile.NamedTemporaryFile(mode='w+') as f:
f.write(yaml.dump(config))
f.flush()
cp = await run(
[
'helm', 'upgrade', release_name, chart_path, '-n',
get_namespace(release_name), '-f', f.name
],
check=False,
)
if cp.returncode:
raise CallError(
f'Failed to upgrade chart release to {new_version!r}: {cp.stderr.decode()}'
)
job.set_progress(100, 'Upgrade complete for chart release')
chart_release = await self.middleware.call(
'chart.release.get_instance', release_name)
self.middleware.send_event('chart.release.query',
'CHANGED',
id=release_name,
fields=chart_release)
await self.chart_release_update_check(
catalog['trains'][release['catalog_train']][chart], chart_release)
if options['update_container_images']:
container_update_job = await self.middleware.call(
'chart.release.pull_container_images', release_name)
await job.wrap(container_update_job)
return chart_release
for particle in ["Pi", "P", "K", "Mu"]:
for varname in ["Brunel_P", "Brunel_PT", "nTracks_Brunel"]:
config = {}
config["sampleVersion" ] = data
config["magnetPolarity" ] = magtype
config["particleName" ] = particle
config["priorCut" ] = cut[particle]
config["pidCut" ] = pidcut[particle]
config["varName" ] = varname
config["outputFile" ] = "binoutput/binning-"+data+".py"
config["minimum" ] = varminmax[varname][0]
config["maximum" ] = varminmax[varname][1]
config["minimumBinWidth" ] = varminmax[varname][2]
config["delta" ] = 1
config["nSigma" ] = 5
config["schemeName" ] = 'binning-'+particle+'-'+data+'-'+magtype
config["numberOfInitialBins" ] = 100
config["startWithIsopopulatedBins" ] = True
config["minRun" ] = None
config["maxRun" ] = None
config["maxFiles" ] = 100 #used
config["mergeBelow" ] = -1000000000
config["mergeAbove" ] = 1000000000
#print(config)
if not os.path.isdir('config_files'): os.mkdir('config_files')
with open("config_files/config-"+particle+"-"+data+"-"+magtype+"-"+varname+".yml", "w") as outfile: yaml.dump(config, outfile, default_flow_style=False)
command = "bash "+UraniaDir+"run python "+UraniaDir+"PIDCalib/PIDPerfScripts/scripts/python/BinningOptimizer/binningPID.py "+"config_files/config-"+particle+"-"+data+"-"+magtype+"-"+varname+".yml"
print(command)
os.system(command)
def scvmGen(pn_nic=None,
pn_ip=None,
pn_prefix=24,
cn_nic=None,
cn_ip=None,
cn_prefix=24,
master=False):
with open(f'{tmpdir}/network-config.mgmt', 'rt') as f:
yam = yaml.load(f)
yam['network']['config'].append({
'mtu':
9000,
'name':
pn_nic,
'subnets': [{
'address': f'{pn_ip}/{pn_prefix}',
'type': 'static'
}],
'type':
'physical'
})
yam['network']['config'].append({
'mtu':
9000,
'name':
cn_nic,
'subnets': [{
'address': f'{cn_ip}/{cn_prefix}',
'type': 'static'
}],
'type':
'physical'
})
with open(f'{tmpdir}/network-config', 'wt') as f:
f.write(yaml.dump(yam))
with open(f'{tmpdir}/user-data', 'rt') as f:
yam2 = yaml.load(f)
yam2['bootcmd'] = [[
'/usr/bin/systemctl', 'enable', '--now', 'cockpit.socket'
], ['/usr/bin/systemctl', 'enable', '--now', 'cockpit.service']]
# if master:
# yam2['bootcmd'].append(
# [f'/usr/bin/script', '-c', '/root/bootstrap.sh', 'bootstrap.log']
# )
with open(f'{pluginpath}/shell/host/scvm_bootstrap.sh',
'rt') as bootstrapfile:
bootstrap = bootstrapfile.read()
yam2['write_files'].append({
'encoding':
'base64',
'content':
base64.encodebytes(bootstrap.encode()),
'owner':
'root:root',
'path':
'/root/bootstrap.sh',
'permissions':
'0777'
})
with open(f'{pluginpath}/shell/host/ipcorrector', 'rt') as ipcorrectorfile:
ipcorrector = ipcorrectorfile.read()
yam2['write_files'].append({
'encoding':
'base64',
'content':
base64.encodebytes(ipcorrector.encode()),
'owner':
'root:root',
'path':
'/usr/local/bin/ipcorrector',
'permissions':
'0777'
})
with open(f'{tmpdir}/user-data', 'wt') as f:
f.write('#cloud-config\n')
f.write(yaml.dump(yam2).replace("\n\n", "\n"))
return json.dumps(indent=4,
obj=json.loads(createReturn(code=200, val=yam)))
def test_init_compute_log_with_bad_config():
with seven.TemporaryDirectory() as tmpdir_path:
with open(os.path.join(tmpdir_path, 'dagster.yaml'), 'w') as fd:
yaml.dump({'compute_logs': {'garbage': 'flargh'}}, fd, default_flow_style=False)
with pytest.raises(DagsterInvalidConfigError, match='Undefined field "garbage"'):
DagsterInstance.from_ref(InstanceRef.from_dir(tmpdir_path))
def as_yaml(self):
data = self.serialize()
dump = yaml.dump(data, default_flow_style=False)
return dump
# create dict for single winners name and avg score
winner = dict(name=competitor, avg=average)
# add the winner to winners list
winners.append(winner)
# evaluate incoming items if average is bigger than lowest, then update top3scores
elif average > min(top3scores):
top3scores.append(average)
top3scores.remove(min(top3scores))
# remove the lowest name/score from winners list
winners = [i for i in winners if not (i['avg'] < min(top3scores))]
# create dict for single winners name and avg score
winner = dict(name=competitor, avg=average)
# add the winner to winners list
winners.append(winner)
winners = sorted(winners, key=lambda k: k['avg'], reverse=True)
# BREAK
# compile final winning results, with disqualifiers
results = {'winners': winners, 'disqualifications': disqualifications}
# dump results into yaml output
with open('results.yml', 'w') as outfile:
yaml.dump(results, outfile, default_flow_style=False, sort_keys=False)
VERSION_TYPE_MINOR = "minor"
VERSION_TYPE_MAJOR = "major"
KEY_VERSION = "version"
PIP_CONFIG_FILE = "pip_config.yml"
if __name__ == "__main__":
version_type = argv[1]
config = yaml.safe_load(open(PIP_CONFIG_FILE, "r"))
version_numbers = [
int(version_number)
for version_number in config[KEY_VERSION].split(".")
]
if version_type == VERSION_TYPE_PATCH:
version_numbers[2] += 1
elif version_type == VERSION_TYPE_MINOR:
version_numbers[1] += 1
elif version_type == VERSION_TYPE_MAJOR:
version_numbers[0] += 1
else:
raise Exception("Invalid version type: {}".format(version_type))
version_numbers = [
str(version_number) for version_number in version_numbers
]
config[KEY_VERSION] = ".".join(version_numbers)
yaml.dump(config, open(PIP_CONFIG_FILE, "w"))
def genUserFromFile(pubkeyfile: str, privkeyfile: str, hostsfile: str):
# with open('user-data.tmpl', 'rt') as f:
# yam = yaml.load(f)
# tmp_meta = f.read()
# pprint.pprint(yam)
with open(pubkeyfile, 'rt') as f:
pubkey = f.read().strip()
with open(privkeyfile, 'rt') as f:
# privkey = '\\n'.join(f.read().splitlines())
# lines = f.read().splitlines()
# privkey = ''
# for line in lines:
# privkey += line.strip() + "\n"
privkey = f.read()
# privkey = privkey.replace("\n", "")
with open(hostsfile, 'rt') as f:
hosts = f.read()
yam = {
'disable_root':
0,
'ssh_pwauth':
True,
'users': [{
'homedir': '/var/lib/ceph',
'groups': 'sudo',
'lock_passwd': False,
'name': 'ceph',
'plain_text_passwd': 'Ablecloud1!',
'ssh-authorized-keys': [pubkey],
'sudo': ['ALL=(ALL) NOPASSWD:ALL']
}, {
'groups': 'sudo',
'lock_passwd': False,
'name': 'ablecloud',
'plain_text_passwd': 'Ablecloud1!',
'ssh-authorized-keys': [pubkey],
'sudo': ['ALL=(ALL) NOPASSWD:ALL']
}, {
'disable_root': 0,
'ssh_pwauth': True,
'name': 'root',
'plain_text_passwd': 'Ablecloud1!',
'ssh-authorized-keys': [pubkey],
}],
'write_files': [{
'encoding': 'base64',
'content': base64.encodebytes(pubkey.encode()),
'owner': 'root:root',
'path': '/root/.ssh/id_rsa.pub',
'permissions': '0644'
}, {
'encoding': 'base64',
'content': base64.encodebytes(privkey.encode()),
'owner': 'root:root',
'path': '/root/.ssh/id_rsa',
'permissions': '0600'
}, {
'encoding': 'base64',
'content': base64.encodebytes(pubkey.encode()),
'owner': 'ceph:ceph',
'path': '/var/lib/ceph/.ssh/id_rsa.pub',
'permissions': '0644'
}, {
'encoding': 'base64',
'content': base64.encodebytes(privkey.encode()),
'owner': 'ceph:ceph',
'path': '/var/lib/ceph/.ssh/id_rsa',
'permissions': '0600'
}, {
'encoding': 'base64',
'content': base64.encodebytes(pubkey.encode()),
'owner': 'ablecloud:ablecloud',
'path': '/home/ablecloud/.ssh/id_rsa.pub',
'permissions': '0644'
}, {
'encoding': 'base64',
'content': base64.encodebytes(privkey.encode()),
'owner': 'ablecloud:ablecloud',
'path': '/home/ablecloud/.ssh/id_rsa',
'permissions': '0600'
}, {
'encoding': 'base64',
'content': base64.encodebytes(hosts.encode()),
'owner': 'root:root',
'path': '/etc/hosts',
'permissions': '0644'
}]
}
# base64.decodebytes(base64.encodebytes(pubkey.encode())).decode()
# with open('user-data', 'wt') as f:
with open(f'{tmpdir}/user-data', 'wt') as f:
f.write('#cloud-config\n')
f.write(yaml.dump(yam).replace("\n\n", "\n"))
exit_status = 1
if is_good_status(last_status):
current_status = 'Failure'
else:
current_status = 'Still Failing'
else:
exit_status = 0
if is_good_status(last_status):
current_status = 'Successful'
else:
current_status = 'Fixed'
category_dict_status[scenario] = current_status
with open(args.build_status_file, 'w') as f:
print('Writing output file: %s' % args.build_status_file)
yaml.dump(category_dict_status, f)
# last successful commit
if (current_status
in ('Successful', 'Fixed')) and trex_last_commit_hash and len(
jobs_list) > 0 and scenario == 'nightly':
with open(args.last_passed_commit, 'w') as f:
print('Writing output file: %s' % args.last_passed_commit)
try_write(f, trex_last_commit_hash)
# mail title
mailtitle_output = scenario.capitalize()
if branch_name:
mailtitle_output += ' (%s)' % branch_name
if build_id:
mailtitle_output += ' - Build #%s' % build_id
def yaml_dump(data, fp, **kwargs):
keys = filter_keys(kwargs.keys(), "safe")
if kwargs.get("safe", False):
return yaml.safe_dump(data, fp, **Base.mk_opt_args(keys, kwargs))
else:
return yaml.dump(data, fp, **kwargs)
def dump_to_yaml(self):
open("./config.yml", "w").write(
yaml.dump({
"token": self.token,
"username": self.user_name
}))
def service_get(ctx, network, name):
"""
Get details about a service in this profile.
"""
# See
# https://stackoverflow.com/questions/16782112/can-pyyaml-dump-dict-items-in-non-alphabetical-order
def represent_ordereddict(dumper, data):
value = []
for item_key, item_value in data.items():
node_key = dumper.represent_data(item_key)
node_value = dumper.represent_data(item_value)
value.append((node_key, node_value))
return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
yaml.add_representer(OrderedDict, represent_ordereddict)
def get_paths_info_for_service(service):
paths = ctx.obj['CLIENT'].paths.list()
has_access_to = ["default-all-outgoing-allowed"]
is_accessible_from = []
for path in paths:
if path.network.name != service.network.name:
continue
if path.destination.name == service.name:
if path.source.name:
is_accessible_from.append(
"%s:%s:%s" %
(path.network.name, path.source.name, path.port))
else:
cidr_blocks = [
subnetwork.cidr_block
for subnetwork in path.source.subnetworks
]
cidr_blocks_string = ",".join(cidr_blocks)
is_accessible_from.append(
"external:%s:%s" % (cidr_blocks_string, path.port))
elif path.source.name == service.name:
has_access_to.append(
"%s:%s:%s" %
(path.network.name, path.destination.name, path.port))
return {
"has_access_to": has_access_to,
"is_accessible_from": is_accessible_from
}
service = get_service_for_cli(ctx, network, name)
paths_info = get_paths_info_for_service(service)
service_info = OrderedDict()
service_info['name'] = service.name
service_info['has_access_to'] = paths_info['has_access_to']
service_info['is_accessible_from'] = paths_info['is_accessible_from']
network_info = OrderedDict()
network_info['name'] = service.network.name
network_info['id'] = service.network.network_id
network_info['block'] = service.network.cidr_block
network_info['region'] = service.network.region
network_info['subnetworks'] = []
service_info['network'] = network_info
for subnetwork in service.subnetworks:
subnetwork_info = OrderedDict()
subnetwork_info['name'] = subnetwork.name
subnetwork_info['id'] = subnetwork.subnetwork_id
subnetwork_info['block'] = subnetwork.cidr_block
subnetwork_info['region'] = subnetwork.region
subnetwork_info['availability_zone'] = subnetwork.availability_zone
subnetwork_info['instances'] = []
for instance in subnetwork.instances:
instance_info = OrderedDict()
instance_info['id'] = instance.instance_id
instance_info['public_ip'] = instance.public_ip
instance_info['private_ip'] = instance.private_ip
instance_info['state'] = instance.state
instance_info['availability_zone'] = instance.availability_zone
subnetwork_info["instances"].append(instance_info)
service_info["network"]["subnetworks"].append(subnetwork_info)
click.echo(yaml.dump(service_info, default_flow_style=False))