def do_list(self):
"""
Usage: list
"""
archives = self.cache._for_adf('archives')
if len(archives):
for n, archive in enumerate(archives.iteritems()):
archive = archive[1]
status = "LOCAL"
cert = ""
if 'signature' in archive:
status = "COMPLETE"
cert = archive['signature'].aid
elif 'links' in archive and archive['links'].upload:
status = "UPLOADED"
if self.verbose:
cert = archive['links'].upload
title = archive['archive'].title
size = archive_size(archive['archive'])
print "{:03d} {:>6} {:>20} {:>10} {:>10}".format(
n+1, size, title, status, cert)
if self.debug > 2:
for doc in archive.itervalues():
pyaml.dump(doc, sys.stdout)
print
else:
print "No available archives."
def writeConfig(self):
log.info("Writing config...")
trigs = {}
for i in self.triggers:
trigs[i.trigger] = i.text
log.info("Saving : {}".format(trigs))
settings = {
"bot": {
"name": self.botname,
"version": self.bot_version,
"msg_prefix": self.bot_prefix,
"cmd_prefix": self.cmd_prefix,
"debug_logging": False
},
"modules": {
"path": self.path,
"load": self.init_modules
},
"triggers": trigs,
"admins": self.admins,
"ai":{"model_directory":"models"}
}
log.info(pyaml.dump(settings))
with open("discord-config.conf", "w") as f:
f.write(pyaml.dump(settings))
log.info("Written")
def test_str_style_pick(self):
a = pyaml.dump(data_str_multiline)
b = pyaml.dump(data_str_multiline, string_val_style='|')
self.assertEqual(a, b)
b = pyaml.dump(data_str_multiline, string_val_style='plain')
self.assertNotEqual(a, b)
self.assertTrue(pyaml.dump('waka waka', string_val_style='|').startswith('|-\n'))
self.assertEqual(pyaml.dump(dict(a=1), string_val_style='|'), 'a: 1\n')
def write_settings_file(self, eps_values):
eps_values['min_samples'] = self.min_samples
eps_values['only_core_samples'] = self.only_core_samples
settingspath = self.pm.blotchfile.parent / 'clustering_settings.yaml'
settingspath.parent.mkdir(exist_ok=True, parents=True)
logger.info("Writing settings file at %s", str(settingspath))
with open(settingspath, 'w') as fp:
pyaml.dump(eps_values, fp)
def test_str_style_pick(self):
a = pyaml.dump(data_str_multiline)
b = pyaml.dump(data_str_multiline, string_val_style="|")
self.assertEqual(a, b)
b = pyaml.dump(data_str_multiline, string_val_style="plain")
self.assertNotEqual(a, b)
self.assertTrue(pyaml.dump("waka waka", string_val_style="|").startswith("|-\n"))
self.assertEqual(pyaml.dump(dict(a=1), string_val_style="|"), "a: 1\n")
def type_diff(a, b):
s = StringIO.StringIO()
pyaml.dump(a, s, safe = True)
a = s.getvalue().splitlines()
s = StringIO.StringIO()
pyaml.dump(b, s, safe = True)
b = s.getvalue().splitlines()
return '\n'.join(difflib.Differ().compare(a, b))
def test_colons_in_strings(self):
val1 = {'foo': ['bar:', 'baz', 'bar:bazzo', 'a: b'], 'foo:': 'yak:'}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
def test_colons_in_strings(self):
val1 = {"foo": ["bar:", "baz", "bar:bazzo", "a: b"], "foo:": "yak:"}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
def test_empty_strings(self):
val1 = {"key": ["", "stuff", "", "more"], "": "value", "k3": ""}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
def test_empty_strings(self):
val1 = {'key': ['', 'stuff', '', 'more'], '': 'value', 'k3': ''}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
def dump(self, content, filepath, indent=4):
"""
Dump settings content to filepath.
Args:
content (str): Settings content.
filepath (str): Settings file location.
"""
with open(filepath, 'w') as fp:
pyaml.dump(content, dst=fp, indent=indent)
def dump(self, data, path):
"""dump data to file
"""
with open(path, "w") as f:
if isinstance(data, MergeableDict):
pyaml.dump(data.convert_to(data), f)
else:
pyaml.dump(data, f)
def main(argv=None):
import argparse
parser = argparse.ArgumentParser(
description='Process and dump prettified YAML to stdout.')
parser.add_argument('path', nargs='?', metavar='path',
help='Path to YAML to read (default: use stdin).')
opts = parser.parse_args(argv or sys.argv[1:])
src = open(opts.path) if opts.path else sys.stdin
try: data = yaml.load(src)
finally: src.close()
pyaml.dump(data, sys.stdout)
def do_list(self):
"""
Usage: list
"""
archives = list(self.cache._for_adf('archives').iteritems())
if len(archives):
ui.print_archives_header()
bydate = sorted(archives,
key=compose(creation, operator.itemgetter(1)))
for n, docs in enumerate(bydate):
fname = docs[0]
archive = docs[1]
status = self.cache.archive_status(fname, archive)
cert = ""
if status == ArchiveStatus.Completed:
status = "COMPLETE"
cert = archive['signature'].aid
elif status == ArchiveStatus.InProgress:
status = "IN PROGRESS"
if self.verbose:
cert = archive['links'].upload
elif status == ArchiveStatus.Failed:
status = "FAILED"
if self.verbose:
cert = archive['links'].upload
elif status == ArchiveStatus.Local:
status = "LOCAL"
elif status == ArchiveStatus.Paused:
status = "PAUSED"
else:
status = "UNKNOWN"
title = archive['archive'].title
size = archive_size(archive['archive'])
ui.print_archives_line(archive={
'num': n+1,
'size': size,
'title': title,
'status': status,
'cert': cert,
'created': archive['archive'].meta.created,
'capsule': archive_capsule(archive) or '-'
})
if self.debug > 2:
for doc in archive.itervalues():
pyaml.dump(doc, sys.stdout)
print
print
else:
print "No available archives."
def test_multiple_docs(self):
docs = [yaml.safe_load(large_yaml), dict(a=1, b=2, c=3)]
docs_str = pyaml.dump_all(docs, vspacing=[3, 2])
self.assertTrue(docs_str.startswith('---'))
self.assertIn('---\n\n\n\na: 1\n\n\n\nb: 2\n\n\n\nc: 3\n', docs_str)
docs_str2 = pyaml.dump(docs, vspacing=[3, 2], multiple_docs=True)
self.assertEqual(docs_str, docs_str2)
docs_str2 = pyaml.dump(docs, vspacing=[3, 2])
self.assertNotEqual(docs_str, docs_str2)
docs_str2 = pyaml.dump_all(docs, explicit_start=False)
self.assertFalse(docs_str2.startswith('---'))
self.assertNotEqual(docs_str, docs_str2)
docs_str = pyaml.dump(docs, multiple_docs=True, explicit_start=False)
self.assertEqual(docs_str, docs_str2)
def writeConfig(self):
"""Save the config, all currently loaded modules will be saved to init_modules"""
self.log.info("Writing config...")
#Add all of our triggers to a nice little dictionary,
#So we can write it to a file
trigs = {}
for i in self.triggers:
trigs[i.trigger] = i.send_text
settings = {
"bot" : {
"name": self.bot_name,
"version": self.bot_version,
"msg_prefix": self.msg_prefix,
"cmd_prefix": self.command_prefix,
"debug_logging": False
},
"modules" : {
"path": self.module_path,
"load": self.loadedModules
},
"triggers": trigs,
"ai": {
"model_directory": "models"
},
"admins": self.admins
}
self.log.info(settings)
#Write the config in YAML format
with open("command-proc.conf", "w") as f:
f.write(pyaml.dump(settings))
self.log.info("Written")
def test_pyyaml_params(self):
d = {'foo': 'lorem ipsum ' * 30} # 300+ chars
for w in 40, 80, 200:
lines = pyaml.dump(d, width=w, indent=10).splitlines()
for n, line in enumerate(lines, 1):
self.assertLess(len(line), w*1.2)
if n != len(lines): self.assertGreater(len(line), w*0.8)
def to_yaml(self):
"""overwrite default DeclarativeBase.to_yaml method because we need to convert MutableDicts first
"""
node_dict = self.to_dict()
node_dict["attrs"] = dict(node_dict["attrs"])
node_dict["system_attrs"] = dict(node_dict["system_attrs"])
return pyaml.dump(node_dict)
def write(): # pragma: no cover
global _cache
yfn = yaml_file_name()
touch(yfn)
with open(yfn, "w") as outfile:
outfile.write(dump(_cache, vspacing=[1, 0]))
return None
def test_print(self): self.assertIs(pyaml.print, pyaml.pprint) self.assertIs(pyaml.print, pyaml.p) buff = io.BytesIO() b = pyaml.dump(data_str_multiline, dst=bytes) pyaml.print(data_str_multiline, file=buff) self.assertEqual(b, buff.getvalue())
def write(self):
d = self._serialize()
s = pyaml.dump(d)
p = os.path.dirname(self.filename)
if not os.path.exists(p):
os.makedirs(p)
with open(self.filename, 'w') as f:
f.write(s)
self._reset_changes()
def output_ja_text(data, wordtypes):
textdata = filter(data)
t = Tokenizer()
tokens = t.tokenize(textdata)
words = sorted([token.surface
for token in tokens
if token.part_of_speech.split(',')[0] in wordtypes])
dictionary = count_words(words)
return pyaml.dump(dictionary, sys.stdout, vspacing=[0, 1])
def test_single_dash_strings(self):
strip_seq_dash = lambda line: line.lstrip().lstrip("-").lstrip()
val1 = {"key": ["-", "-stuff", "- -", "- more-", "more-", "--"]}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
val1_str_lines = val1_str.splitlines()
self.assertEqual(strip_seq_dash(val1_str_lines[2]), "-stuff")
self.assertEqual(strip_seq_dash(val1_str_lines[5]), "more-")
self.assertEqual(strip_seq_dash(val1_str_lines[6]), "--")
val1 = {"key": "-"}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
def test_single_dash_strings(self):
strip_seq_dash = lambda line: line.lstrip().lstrip('-').lstrip()
val1 = {'key': ['-', '-stuff', '- -', '- more-', 'more-', '--']}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
self.assertEqual(val1, val2)
self.assertEqual(val1_str, val2_str)
self.assertEqual(val2, val3)
val1_str_lines = val1_str.splitlines()
self.assertEqual(strip_seq_dash(val1_str_lines[2]), '-stuff')
self.assertEqual(strip_seq_dash(val1_str_lines[5]), 'more-')
self.assertEqual(strip_seq_dash(val1_str_lines[6]), '--')
val1 = {'key': '-'}
val1_str = pyaml.dump(val1)
val2 = yaml.safe_load(val1_str)
val2_str = pyaml.dump(val2)
val3 = yaml.safe_load(val2_str)
def test_vspacing(self):
data = yaml.safe_load(large_yaml)
a = self.flatten(data)
b = pyaml.dump(data, unicode, vspacing=[2, 1])
self.assertEqual(a, self.flatten(yaml.safe_load(b)))
pos, pos_list = 0, list()
while True:
pos = b.find(u'\n', pos+1)
if pos < 0: break
pos_list.append(pos)
self.assertEqual( pos_list,
[ 12, 13, 25, 33, 53, 74, 89, 108, 158, 185, 265, 300, 345, 346, 356, 376, 400, 426, 427,
460, 461, 462, 470, 508, 564, 603, 604, 605, 611, 612, 665, 666, 690, 691, 715, 748,
777, 806, 807, 808, 817, 818, 832, 843, 878, 948, 949, 961, 974, 1009, 1032, 1052,
1083, 1102, 1123, 1173, 1195, 1234, 1257, 1276, 1300, 1301, 1312, 1325, 1341, 1359,
1374, 1375, 1383, 1397, 1413, 1431, 1432, 1453, 1454, 1467, 1468, 1485, 1486, 1487,
1498, 1499, 1530, 1531, 1551, 1552, 1566, 1577, 1590, 1591, 1612, 1613, 1614, 1622,
1623, 1638, 1648, 1649, 1657, 1658, 1688, 1689, 1698, 1720, 1730 ] )
b = pyaml.dump(data, unicode)
self.assertNotIn('\n\n', b)
def do_list(self):
"""
Usage: list
"""
certs = self.cache._for_adf('certs')
if len(certs):
for n, cert in enumerate(certs.iteritems()):
cert = cert[1]
aid = cert['signature'].aid
title = cert['archive'].title
size = archive_size(cert['archive'])
print "{:>10} {:>6} {:<}".format(
aid, size, title)
if self.debug > 2:
for doc in cert.itervalues():
pyaml.dump(doc, sys.stdout)
print
else:
print "No available certificates."
def write_file(path, data, token):
try:
branch = local_repo.create_branch(token, get_latest_commit())
except Exception as e:
# TODO more specific exception
branch = repo.lookup_branch('master')
author = committer = Signature('Joe S', '*****@*****.**')
print(path)
print(pretty_yaml.dump(data, string_val_style='>'))
return token
def test_monkey_patch(self):
d = {'a': {'b': 'c'}}
# test default
pyaml.dump(d)
self.assertFalse(pyaml.patched_values['safe'])
self.assertFalse(pyaml.patched_values['force_embed'])
self.assertTrue(pyaml.patched_values['patched'])
self.assertTrue(hasattr(yaml.serializer.Serializer, 'orig_anchor_node'))
self.assertTrue(hasattr(yaml.serializer.Serializer, 'orig_serialize_node'))
self.assertTrue(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence'))
self.assertTrue(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence_item'))
# test resetting
pyaml.dump(d, safe=True)
self.assertTrue(pyaml.patched_values['safe'])
self.assertFalse(pyaml.patched_values['force_embed'])
self.assertTrue(pyaml.patched_values['patched'])
self.assertFalse(hasattr(yaml.serializer.Serializer, 'orig_anchor_node'))
self.assertFalse(hasattr(yaml.serializer.Serializer, 'orig_serialize_node'))
self.assertFalse(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence'))
self.assertFalse(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence_item'))
# test patching again
pyaml.dump(d, force_embed=True)
self.assertFalse(pyaml.patched_values['safe'])
self.assertTrue(pyaml.patched_values['force_embed'])
self.assertTrue(pyaml.patched_values['patched'])
self.assertTrue(hasattr(yaml.serializer.Serializer, 'orig_anchor_node'))
self.assertTrue(hasattr(yaml.serializer.Serializer, 'orig_serialize_node'))
self.assertTrue(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence'))
self.assertTrue(hasattr(yaml.emitter.Emitter, 'orig_expect_block_sequence_item'))
def _print_dict_ip(d, header=None, format='table'):
if format == "json":
return json.dumps(d, indent=4)
elif format == "yaml":
return pyaml.dump(d)
elif format == "table":
return dict_printer(d,
order=["network",
"version",
"addr"],
output="table",
sort_keys=True)
else:
return d
def yaml(self):
logs_list = None
if self.logs is not None:
logs_list = [lm.yaml() for lm in self.logs]
data = OrderedDict([
('service_name', self.name),
('team_name', self.team_name),
('port', self.port),
('healthcheck_url', self.healthcheck_url),
('logs', logs_list),
('environments', [e.yaml() for e in self.environments])
])
return pyaml.dump(OrderedDict((k, v) for k, v in data.items() if v is not None))
def convertGraph(self, graph):
"""Convert graph."""
cisgraph = fbpToCis(graph['content'])
cisgraph = as_str(cisgraph, recurse=True, allow_pass=True)
# Write to temp file and validate
tmpfile = tempfile.NamedTemporaryFile(suffix="yml", prefix="cis",
delete=False)
yaml.safe_dump(cisgraph, tmpfile, default_flow_style=False)
yml_prep = prep_yaml(tmpfile.name)
os.remove(tmpfile.name)
s = get_schema()
yml_norm = s.normalize(yml_prep)
try:
s.validate(yml_norm)
except BaseException as e:
print(e)
raise RestException('Invalid graph %s', 400, e)
self.setRawResponse()
return pyaml.dump(cisgraph)
def auth(self):
status = True
try:
config_file, filename = tempfile.mkstemp()
config_content = {
'apiVersion':
'v1',
'clusters': [{
'cluster': self.metadata['cluster'],
'name': self.name,
}],
'contexts': [{
'context': {
'cluster': self.name,
'user': self.name,
},
'name': self.name,
}],
'current-context':
self.name,
'kind':
'Config',
'preferences': {},
'users': [{
'name': self.name,
'user': self.metadata['user']
}]
}
os.write(config_file, pyaml.dump(config_content).encode())
os.close(config_file)
self.config_wrapper = pykube.KubeConfig.from_file(filename)
os.remove(filename)
self.api = pykube.HTTPClient(self.config_wrapper)
except URLError as exception:
logger.error(exception)
status = False
return status
def to_yaml(value):
"""
Output value as YAML string
"""
return pyaml.dump(value)
def map(function, input_list):
"""
map operation of Couler
"""
# Enforce the function to run and lock to add into step
if callable(function):
states._update_steps_lock = False
# TODO (terrytangyuan): Support functions with multiple arguments.
para = input_list[0]
inner = function(para)
if inner is None:
raise SyntaxError("require function return value")
states._update_steps_lock = True
else:
raise TypeError("require loop over a function to run")
inner_dict = output.extract_step_return(inner)
template_name = inner_dict["name"]
inner_step = Step(name=inner_dict["id"], template=template_name)
parameters = []
items_param_name = "%s-para-name" % template_name
items_param_dict = {"name": items_param_name}
function_template = states.workflow.get_template(template_name)
function_template_dict = function_template.to_dict()
if "resource" in function_template_dict:
# Update the template with the new dynamic `metadata.name`.
manifest_dict = yaml.safe_load(
function_template_dict["resource"]["manifest"]
)
manifest_dict["metadata"]["name"] = (
"'{{inputs.parameters.%s}}'" % items_param_name
)
function_template = states.workflow.get_template(template_name)
function_template.manifest = pyaml.dump(manifest_dict)
# Append this items parameter to input parameters in the template
function_template.args.append(items_param_dict)
states.workflow.add_template(function_template)
input_parameters = [items_param_dict]
else:
input_parameters = function_template_dict["inputs"]["parameters"]
for para_name in input_parameters:
parameters.append(
{
"name": para_name["name"],
"value": '"{{item.%s}}"' % para_name["name"],
}
)
inner_step.arguments = {"parameters": parameters}
with_items = []
for para_values in input_list:
item = {}
if not isinstance(para_values, list):
para_values = [para_values]
for j in range(len(input_parameters)):
para_name = input_parameters[j]["name"]
item[para_name] = para_values[j]
with_items.append(item)
inner_step.with_items = with_items
states.workflow.add_step(inner_dict["id"], inner_step)
return inner_step
def run_job(
manifest,
success_condition,
failure_condition,
timeout=None,
retry=None,
step_name=None,
pool=None,
env=None,
set_owner_reference=True,
):
"""
Create a k8s job. For example, the pi-tmpl template in
https://github.com/argoproj/argo/blob/master/examples/k8s-jobs.yaml
:param manifest: YAML specification of the job to be created.
:param success_condition: expression for verifying job success.
:param failure_condition: expression for verifying job failure.
:param timeout: To limit the elapsed time for a workflow in seconds.
:param step_name: is only used while developing functions of step zoo.
:param env: environmental parameter with a dict types, e.g., {"OS_ENV_1": "OS_ENV_value"} # noqa: E501
:param set_owner_reference: Whether to set the workflow as the job's owner reference.
If `True`, the job will be deleted once the workflow is deleted.
:return: output
"""
if manifest is None:
raise ValueError("Input manifest can not be null")
func_name, caller_line = utils.invocation_location()
func_name = (utils.argo_safe_name(step_name)
if step_name is not None else func_name)
args = []
if states.workflow.get_template(func_name) is None:
if states._outputs_tmp is not None and env is not None:
env["inferred_outputs"] = states._outputs_tmp
# Generate the inputs for the manifest template
envs, parameters, args = utils.generate_parameters_run_job(env)
# update the env
if env is not None:
manifest_dict = yaml.safe_load(manifest)
manifest_dict["spec"]["env"] = envs
# TODO this is used to pass the test cases,
# should be fixed in a better way
if ("labels" in manifest_dict["metadata"] and "argo.step.owner"
in manifest_dict["metadata"]["labels"]):
manifest_dict["metadata"]["labels"][
"argo.step.owner"] = "'{{pod.name}}'"
manifest = pyaml.dump(manifest_dict)
template = Job(
name=func_name,
args=args,
action="create",
manifest=manifest,
set_owner_reference=set_owner_reference,
success_condition=success_condition,
failure_condition=failure_condition,
timeout=timeout,
retry=retry,
pool=pool,
)
states.workflow.add_template(template)
step_name = step_update_utils.update_step(func_name, args, step_name,
caller_line)
# return job name and job uid for reference
rets = _job_output(step_name, func_name)
states._steps_outputs[step_name] = rets
pb_step = proto_repr.step_repr( # noqa: F841
step_name=step_name,
tmpl_name=func_name,
image=None,
source=None,
script_output=None,
input=None,
output=rets,
manifest=manifest,
success_cond=success_condition,
failure_cond=failure_condition,
)
return rets
def train(
image=None,
command="",
secret=None,
no_chief=True,
chief_image=None,
chief_resources=None,
chief_restart_policy="Never",
chief_command=None,
num_ps=0,
ps_image=None,
ps_resources=None,
ps_restart_policy="Never",
ps_command=None,
num_workers=0,
worker_image=None,
worker_resources=None,
worker_restart_policy="Never",
worker_command=None,
clean_pod_policy="Running",
timeout=None,
):
name = "tf-train-%s" % str(uuid.uuid4())
success_condition = ("status.replicaStatuses.Worker.succeeded == %s" %
num_workers)
failure_condition = "status.replicaStatuses.Worker.failed > 0"
manifest = copy.deepcopy(manifest_template)
manifest["metadata"].update({"name": name})
manifest["spec"].update({"cleanPodPolicy": clean_pod_policy})
if not no_chief:
chief_image = chief_image if chief_image else image
chief_command = chief_command if chief_command else command
chief_pod = _generate_pod_spec(
pod_template,
container_template,
allowed_pod_types=pod_types,
pod_type="Chief",
image=chief_image,
replicas=1,
secret=secret,
command=chief_command,
resources=chief_resources,
restart_policy=chief_restart_policy,
)
manifest["spec"]["tfReplicaSpecs"].update({"Chief": chief_pod})
if num_ps > 0:
ps_image = ps_image if ps_image else image
ps_command = ps_command if ps_command else command
ps_pod = _generate_pod_spec(
pod_template,
container_template,
allowed_pod_types=pod_types,
pod_type="PS",
image=ps_image,
replicas=num_ps,
secret=secret,
command=ps_command,
resources=ps_resources,
restart_policy=ps_restart_policy,
)
manifest["spec"]["tfReplicaSpecs"].update({"PS": ps_pod})
if num_workers > 0:
worker_image = worker_image if worker_image else image
worker_command = worker_command if worker_command else command
worker_pod = _generate_pod_spec(
pod_template,
container_template,
allowed_pod_types=pod_types,
pod_type="Worker",
image=worker_image,
replicas=num_workers,
secret=secret,
command=worker_command,
resources=worker_resources,
restart_policy=worker_restart_policy,
)
manifest["spec"]["tfReplicaSpecs"].update({"Worker": worker_pod})
step_name, _ = utils.invocation_location()
couler.run_job(
manifest=pyaml.dump(manifest),
success_condition=success_condition,
failure_condition=failure_condition,
step_name=step_name,
timeout=timeout,
)
def test_print_args(self):
buff = io.BytesIO()
args = 1, 2, 3
b = pyaml.dump(args, dst=bytes)
pyaml.print(*args, file=buff)
self.assertEqual(b, buff.getvalue())
def test_force_embed(self):
b = pyaml.dump(data, unicode, force_embed=True)
c = pyaml.dump(data, unicode, safe=True, force_embed=True)
for char, dump in it.product('*&', [b, c]):
self.assertNotIn(char, dump)
def test_simple(self):
a = self.flatten(data)
b = pyaml.dump(data, unicode)
self.assertEqual(a, self.flatten(yaml.safe_load(b)))
model.load_state_dict(torch.load(experiment.model.state_dict))
model.to(experiment.session.device)
# Optimizer
optimizer: torch.optim.Optimizer = import_(
experiment.optimizer.fn)(model.parameters(), *experiment.optimizer.args,
**experiment.optimizer.kwargs)
if 'state_dict' in experiment.optimizer:
optimizer.load_state_dict(torch.load(experiment.optimizer.state_dict))
# Logger
if len(experiment.session.log.when) > 0:
logger = SummaryWriter(experiment.session.log.folder)
logger.add_text(
'Experiment',
textwrap.indent(pyaml.dump(experiment, safe=True, sort_dicts=False),
' '), experiment.samples)
else:
logger = None
# Saver
if len(experiment.session.checkpoint.when) > 0:
saver = Saver(experiment.session.checkpoint.folder)
if experiment.epoch == 0:
saver.save_experiment(experiment, suffix=f'e{experiment.epoch:04d}')
else:
saver = None
# endregion
# Datasets and dataloaders
dataset = SolubilityDataset(experiment.session.data.path)
def save(self) -> 'ConfigurationLoader':
with self._config_file.open('w') as fp:
pyaml.dump(self._config, fp)
return self
dataloader_train, dataloader_val = get_dataloaders(ex, session)
session['misc']['samples'] = {
'train': len(dataloader_train.dataset),
'val': len(dataloader_val.dataset),
}
session['misc']['targets'] = {
'train': len(dataloader_train.dataset.datasets),
'val': len(dataloader_val.dataset.datasets),
}
if ex['completed_epochs'] == 0:
saver.save_experiment(ex, epoch=ex['completed_epochs'], samples=ex['samples'])
logger.add_text('Experiment',
textwrap.indent(pyaml.dump(ex, safe=True, sort_dicts=False, force_embed=True), ' '),
global_step=ex['samples'])
# endregion
# region Training
def training_function(trainer, batch: DecoyBatch):
batch = batch.to(session['device'])
results = model(batch)
loss_local_lddt = torch.tensor(0., device=session['device'])
loss_global_gdtts = torch.tensor(0., device=session['device'])
loss_rank_gdtts = torch.tensor(0., device=session['device'])
if ex['loss']['local_lddt']['weight'] > 0:
node_mask = torch.isfinite(batch.lddt)
def train(args):
cuda_availability = torch.cuda.is_available()
print('\n*************************')
print('`CUDA` available: {}'.format(cuda_availability))
print('Device specified: {}'.format(args.device))
print('*************************\n')
# load the config of the trained model:
with open(args.pretrained_output / "train_arguments.yaml") as yaml_data:
pretrain_arguments = yaml.load(yaml_data, Loader=yaml.FullLoader)
pretrained_model = algorithms[pretrain_arguments["alg"]].load(
args.pretrained_output /
"".join(pretrain_arguments["model_name"].split(".")[:-1]),
device='cpu')
# Prepare tensorboard logging
log_name = '{}_{}_{}'.format(args.experiment_name, args.task_name,
datetime.now().strftime('%d-%m_%H-%M-%S'))
run_dir = args.tensorboard_log + "/" + log_name
Path(run_dir).mkdir(parents=True, exist_ok=True)
callbacks = []
# callbacks.append(CheckpointCallback(
# save_freq=1000000, save_path=run_dir, name_prefix='rl_model'))
callbacks.append(LoggingCallback(logpath=run_dir))
train_args = copy.copy(args)
train_args.config = train_args.config.name
pyaml.dump(train_args.__dict__,
open(os.path.join(run_dir, 'train_arguments.yaml'), 'w'))
assert args.task_name == pretrain_arguments[
"task_name"], "Envs must match for transfer learning"
# Create the vectorized environment
n_envs = train_args.n_envs # Number of processes to use
env = make_vec_env(args.task_name, n_envs=n_envs)
# define network architecture
if "GnnPolicy" in args.policy and args.net_arch is not None:
for net_arch_part in args.net_arch.keys():
for i, (layer_class_name,
layer_size) in enumerate(args.net_arch[net_arch_part]):
if hasattr(nn, layer_class_name):
args.net_arch[net_arch_part][i] = (getattr(
nn, layer_class_name), layer_size)
elif hasattr(nerve_net_conv, layer_class_name):
args.net_arch[net_arch_part][i] = (getattr(
nerve_net_conv, layer_class_name), layer_size)
else:
def get_class(x):
return globals()[x]
c = get_class(layer_size)
assert c is not None, f"Unkown layer class '{layer_class_name}'"
args.net_arch[net_arch_part][i] = (c, layer_size)
with open(os.path.join(run_dir, 'net_arch.txt'), 'w') as fp:
fp.write(str(args.net_arch))
# Create the model
alg_class = algorithms[args.alg]
policy_kwargs = dict()
if args.net_arch is not None:
policy_kwargs['net_arch'] = args.net_arch
if args.activation_fn is not None:
policy_kwargs["activation_fn"] = activation_functions[
args.activation_fn]
# policy_kwargs['device'] = args.device if args.device is not None else get_device('auto')
if "GnnPolicy" in args.policy:
policy_kwargs["mlp_extractor_kwargs"] = {
"task_name": args.task_name,
'device': args.device,
'gnn_for_values': args.gnn_for_values,
'controller_option': controller_option[args.controller_option],
'embedding_option': embedding_option[args.embedding_option],
'root_option': root_option[args.root_option],
'drop_body_nodes': args.drop_body_nodes,
'use_sibling_relations': args.use_sibling_relations,
'xml_assets_path': args.xml_assets_path,
'policy_readout_mode': args.policy_readout_mode
}
alg_kwargs = args.__dict__.copy()
alg_kwargs.pop("config", None)
alg_kwargs.pop("task_name", None)
alg_kwargs.pop("policy", None)
alg_kwargs.pop("activation_fn", None)
alg_kwargs.pop("gnn_for_values", None)
alg_kwargs.pop("embedding_option", None)
alg_kwargs.pop("controller_option", None)
alg_kwargs.pop("root_option", None)
alg_kwargs.pop("xml_assets_path", None)
alg_kwargs.pop("alg", None)
alg_kwargs.pop("net_arch", None)
alg_kwargs.pop("experiment_name", None)
alg_kwargs.pop("job_dir", None)
alg_kwargs.pop("total_timesteps", None)
alg_kwargs.pop("model_name", None)
alg_kwargs.pop("n_envs", None)
alg_kwargs.pop("drop_body_nodes", None)
alg_kwargs.pop("use_sibling_relations", None)
alg_kwargs.pop("experiment_name_suffix", None)
alg_kwargs.pop("policy_readout_mode", None)
alg_kwargs.pop("pretrained_output", None)
model = alg_class(
args.policy,
env,
verbose=1,
# n_steps=args.n_steps,
policy_kwargs=policy_kwargs,
# device=args.device,
# tensorboard_log=args.tensorboard_log,
# learning_rate=args.learning_rate,
# batch_size=args.batch_size,
# n_epochs=args.n_epochs,
**alg_kwargs)
# model.learn(total_timesteps=args.total_timesteps,
# callback=callbacks,
# tb_log_name=log_name)
# PPO Learn parameters:
total_timesteps = args.total_timesteps
callback = callbacks
log_interval = 1
eval_env = make_vec_env(args.task_name, n_envs=1)
eval_freq = 1e4
n_eval_episodes = 3
tb_log_name = log_name
eval_log_path = None
reset_num_timesteps = True
#################################
##### Custom Transfer Learn #####
#################################
iteration = 0
total_timesteps, callback = model._setup_learn(
total_timesteps, eval_env, callback, eval_freq, n_eval_episodes,
eval_log_path, reset_num_timesteps, tb_log_name)
### setup pretrained model ###
pretrained_model.num_timesteps = 0
pretrained_model._episode_num = 0
pretrained_model._total_timesteps = total_timesteps
pretrained_model.ep_info_buffer = deque(maxlen=100)
pretrained_model.ep_success_buffer = deque(maxlen=100)
pretrained_model._last_obs = model.env.reset()
pretrained_model._last_dones = np.zeros((model.env.num_envs, ), dtype=bool)
callback.on_training_start(locals(), globals())
while pretrained_model.num_timesteps < total_timesteps:
continue_training = pretrained_model.collect_rollouts(
model.env,
callback,
model.rollout_buffer,
n_rollout_steps=model.n_steps)
if continue_training is False:
break
iteration += 1
model._update_current_progress_remaining(
pretrained_model.num_timesteps, total_timesteps)
# Display training infos
if log_interval is not None and iteration % log_interval == 0:
fps = int(pretrained_model.num_timesteps /
(time.time() - model.start_time))
logger.record("time/iterations", iteration, exclude="tensorboard")
if len(model.ep_info_buffer) > 0 and len(
model.ep_info_buffer[0]) > 0:
logger.record(
"rollout/ep_rew_mean",
safe_mean(
[ep_info["r"] for ep_info in model.ep_info_buffer]))
logger.record(
"rollout/ep_len_mean",
safe_mean(
[ep_info["l"] for ep_info in model.ep_info_buffer]))
logger.record("time/fps", fps)
logger.record("time/time_elapsed",
int(time.time() - model.start_time),
exclude="tensorboard")
logger.record("time/total_timesteps",
pretrained_model.num_timesteps,
exclude="tensorboard")
logger.dump(step=pretrained_model.num_timesteps)
model.train()
callback.on_training_end()
model.save(
os.path.join(args.tensorboard_log + "/" + log_name, args.model_name))
def test_ordereddict(self):
d = OrderedDict((i, '') for i in reversed(range(10)))
lines = pyaml.dump(d).splitlines()
self.assertEqual(lines, list(reversed(sorted(lines))))
def create_docker_compose_cli(jsonData):
###### HardCoded Values ######
# n_orgs = args[1]
# n_peer_org = args[2]
peer_name = "peer"
# TODO: Update to take value from json
network_name = ["byfn"]
with open("./network/template/docker-compose-cli-template.yaml") as f:
list_doc = yaml.load(f)
#### Networks ####
list_doc["version"] = "'" + str(2) + "'"
networks = dict.fromkeys(network_name, )
list_doc["networks"] = networks
#### Volumes ####
## Orderer Volume ##
org_final_name = []
for orderer in jsonData["organizations"]["ordererOrg"]["url"]:
org_final_name.append(orderer)
### ORG Volumes ###
for i in range(0, len(jsonData["organizations"]["peerOrgs"])):
org = jsonData["organizations"]["peerOrgs"][i]["url"]
for peer in range(
0, int(jsonData["organizations"]["peerOrgs"][i]["count"])):
orgName = peer_name + str(peer) + "." + org
org_final_name.append(orgName)
volumes = dict.fromkeys(org_final_name, )
list_doc["volumes"] = volumes
#### Services ####
services = {}
for org in org_final_name:
if not org in services:
services[org] = {}
if not "extends" in services[org]:
services[org]["extends"] = {}
if not "service" in services[org]["extends"]:
services[org]["extends"]["service"] = {}
if not "file" in services[org]["extends"]:
services[org]["extends"]["file"] = {}
if not "container_name" in services[org]:
services[org]["container_name"] = {}
if not "networks" in services[org]:
services[org]["networks"] = {}
services[org]["extends"]["service"] = org
services[org]["extends"]["file"] = "base/docker-compose-base.yaml"
services[org]["container_name"] = org
services[org]["networks"] = network_name
list_doc["services"].update(services)
#### Services CLI ####
for i in range(0, len(list_doc["services"]["cli"]["environment"])):
env = list_doc["services"]["cli"]["environment"][i]
if list_doc["services"]["cli"]["environment"][i].find("org1") != -1:
env = env.replace("org1",
jsonData["organizations"]["peerOrgs"][0]["name"])
if list_doc["services"]["cli"]["environment"][i].find("Org1MSP") != -1:
env = env.replace(
"Org1MSP", jsonData["organizations"]["peerOrgs"][0]["mspID"])
list_doc["services"]["cli"]["environment"][i] = env
list_doc["services"]["cli"]["depends_on"] = org_final_name
list_doc["services"]["cli"]["networks"] = network_name
# print (list_doc["services"])
with open("./network/docker-compose-cli.yaml", "w+") as f:
pyaml.dump(list_doc, f, vspacing=[2, 1])
def train(
image=None,
command="",
secret=None,
master_image=None,
master_resources=None,
master_restart_policy="Never",
master_command=None,
num_workers=0,
worker_image=None,
worker_resources=None,
worker_restart_policy="Never",
worker_command=None,
clean_pod_policy="Running",
timeout=None,
):
name = "pytorch-train-%s" % str(uuid.uuid4())
success_condition = "status.pytorchReplicaStatuses.Worker.succeeded > 0"
failure_condition = "status.pytorchReplicaStatuses.Worker.failed > 0"
manifest = copy.deepcopy(manifest_template)
manifest["metadata"].update({"name": name})
manifest["spec"].update({"cleanPodPolicy": clean_pod_policy})
master_image = master_image if master_image else image
master_command = master_command if master_command else command
chief_pod = _generate_pod_spec(
pod_template,
container_template,
allowed_pod_types=pod_types,
pod_type="Master",
image=master_image,
replicas=1,
secret=secret,
command=master_command,
resources=master_resources,
restart_policy=master_restart_policy,
)
manifest["spec"]["pytorchReplicaSpecs"].update({"Master": chief_pod})
if num_workers > 0:
worker_image = worker_image if worker_image else image
worker_command = worker_command if worker_command else command
worker_pod = _generate_pod_spec(
pod_template,
container_template,
allowed_pod_types=pod_types,
pod_type="Worker",
image=worker_image,
replicas=num_workers,
secret=secret,
command=worker_command,
resources=worker_resources,
restart_policy=worker_restart_policy,
)
manifest["spec"]["pytorchReplicaSpecs"].update({"Worker": worker_pod})
step_name, _ = utils.invocation_location()
couler.run_job(
manifest=pyaml.dump(manifest),
success_condition=success_condition,
failure_condition=failure_condition,
step_name=step_name,
timeout=timeout,
)
def __init__(self, problems):
message = 'Organization structure is invalid. Problems:\n%s' % pyaml.dump(problems)
Exception.__init__(self, message)
},
"validation": {
"accuracy": val_accuracy,
"loss": val_loss
}
}
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--runpath", type=Path, required=True)
parser.add_argument("--datapath", type=Path, required=True)
parser.add_argument("--batch_size", type=int, required=True)
parser.add_argument("--learning_rate", type=float, required=True)
parser.add_argument("--weight_decay", type=float, required=True)
parser.add_argument("--number_epochs", type=int, required=True)
parser.add_argument("--number_workers", type=int, required=True)
parser.add_argument("--device", type=str, default="cpu")
args = parser.parse_args()
random_hash = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=10))
args.logdir = args.runpath / f"bs{args.batch_size}_lr{args.learning_rate}_wd{args.weight_decay}_{random_hash}"
metric_dictionary = train(args)
config = vars(args)
final_log_dictionary = {"config": config, "results": metric_dictionary}
with open(args.logdir / "final_results.yaml", "w") as outfile:
pyaml.dump(final_log_dictionary, outfile)
def main():
# region Setup
conf = parse_args()
setup_seeds(conf.session.seed)
tb_logger, tb_img_logger, json_logger = setup_all_loggers(conf)
logger.info("Parsed configuration:\n" +
pyaml.dump(OmegaConf.to_container(conf),
safe=True,
sort_dicts=False,
force_embed=True))
# region Predicate classification engines
datasets, dataset_metadata = build_datasets(conf.dataset)
dataloaders = build_dataloaders(conf, datasets)
model = build_model(conf.model,
dataset_metadata["train"]).to(conf.session.device)
criterion = PredicateClassificationCriterion(conf.losses)
pred_class_trainer = Trainer(pred_class_training_step, conf)
pred_class_trainer.model = model
pred_class_trainer.criterion = criterion
pred_class_trainer.optimizer, scheduler = build_optimizer_and_scheduler(
conf.optimizer, pred_class_trainer.model)
pred_class_validator = Validator(pred_class_validation_step, conf)
pred_class_validator.model = model
pred_class_validator.criterion = criterion
pred_class_tester = Validator(pred_class_validation_step, conf)
pred_class_tester.model = model
pred_class_tester.criterion = criterion
# endregion
if "resume" in conf:
checkpoint = Path(conf.resume.checkpoint).expanduser().resolve()
logger.debug(f"Resuming checkpoint from {checkpoint}")
Checkpoint.load_objects(
{
"model": pred_class_trainer.model,
"optimizer": pred_class_trainer.optimizer,
"scheduler": scheduler,
"trainer": pred_class_trainer,
},
checkpoint=torch.load(checkpoint,
map_location=conf.session.device),
)
logger.info(f"Resumed from {checkpoint}, "
f"epoch {pred_class_trainer.state.epoch}, "
f"samples {pred_class_trainer.global_step()}")
# endregion
# region Predicate classification training callbacks
def increment_samples(trainer: Trainer):
images = trainer.state.batch[0]
trainer.state.samples += len(images)
pred_class_trainer.add_event_handler(Events.ITERATION_COMPLETED,
increment_samples)
ProgressBar(persist=True, desc="Pred class train").attach(
pred_class_trainer, output_transform=itemgetter("losses"))
tb_logger.attach(
pred_class_trainer,
OptimizerParamsHandler(
pred_class_trainer.optimizer,
param_name="lr",
tag="z",
global_step_transform=pred_class_trainer.global_step,
),
Events.EPOCH_STARTED,
)
pred_class_trainer.add_event_handler(
Events.ITERATION_COMPLETED,
PredicateClassificationMeanAveragePrecisionBatch())
pred_class_trainer.add_event_handler(Events.ITERATION_COMPLETED,
RecallAtBatch(sizes=(5, 10)))
tb_logger.attach(
pred_class_trainer,
OutputHandler(
"train",
output_transform=lambda o: {
**o["losses"],
"pc/mAP": o["pc/mAP"].mean().item(),
**{k: r.mean().item()
for k, r in o["recalls"].items()},
},
global_step_transform=pred_class_trainer.global_step,
),
Events.ITERATION_COMPLETED,
)
pred_class_trainer.add_event_handler(
Events.EPOCH_COMPLETED,
log_metrics,
"Predicate classification training",
"train",
json_logger=None,
tb_logger=tb_logger,
global_step_fn=pred_class_trainer.global_step,
)
pred_class_trainer.add_event_handler(
Events.EPOCH_COMPLETED,
PredicateClassificationLogger(
grid=(2, 3),
tag="train",
logger=tb_img_logger.writer,
metadata=dataset_metadata["train"],
global_step_fn=pred_class_trainer.global_step,
),
)
tb_logger.attach(
pred_class_trainer,
EpochHandler(
pred_class_trainer,
tag="z",
global_step_transform=pred_class_trainer.global_step,
),
Events.EPOCH_COMPLETED,
)
pred_class_trainer.add_event_handler(
Events.EPOCH_COMPLETED,
lambda _: pred_class_validator.run(dataloaders["val"]))
# endregion
# region Predicate classification validation callbacks
ProgressBar(persist=True,
desc="Pred class val").attach(pred_class_validator)
if conf.losses["bce"]["weight"] > 0:
Average(output_transform=lambda o: o["losses"]["loss/bce"]).attach(
pred_class_validator, "loss/bce")
if conf.losses["rank"]["weight"] > 0:
Average(output_transform=lambda o: o["losses"]["loss/rank"]).attach(
pred_class_validator, "loss/rank")
Average(output_transform=lambda o: o["losses"]["loss/total"]).attach(
pred_class_validator, "loss/total")
PredicateClassificationMeanAveragePrecisionEpoch(
itemgetter("target", "output")).attach(pred_class_validator, "pc/mAP")
RecallAtEpoch((5, 10),
itemgetter("target",
"output")).attach(pred_class_validator,
"pc/recall_at")
pred_class_validator.add_event_handler(
Events.EPOCH_COMPLETED,
lambda val_engine: scheduler.step(val_engine.state.metrics["loss/total"
]),
)
pred_class_validator.add_event_handler(
Events.EPOCH_COMPLETED,
log_metrics,
"Predicate classification validation",
"val",
json_logger,
tb_logger,
pred_class_trainer.global_step,
)
pred_class_validator.add_event_handler(
Events.EPOCH_COMPLETED,
PredicateClassificationLogger(
grid=(2, 3),
tag="val",
logger=tb_img_logger.writer,
metadata=dataset_metadata["val"],
global_step_fn=pred_class_trainer.global_step,
),
)
pred_class_validator.add_event_handler(
Events.COMPLETED,
EarlyStopping(
patience=conf.session.early_stopping.patience,
score_function=lambda val_engine: -val_engine.state.metrics[
"loss/total"],
trainer=pred_class_trainer,
),
)
pred_class_validator.add_event_handler(
Events.COMPLETED,
Checkpoint(
{
"model": pred_class_trainer.model,
"optimizer": pred_class_trainer.optimizer,
"scheduler": scheduler,
"trainer": pred_class_trainer,
},
DiskSaver(
Path(conf.checkpoint.folder).expanduser().resolve() /
conf.fullname),
score_function=lambda val_engine: val_engine.state.metrics[
"pc/recall_at_5"],
score_name="pc_recall_at_5",
n_saved=conf.checkpoint.keep,
global_step_transform=pred_class_trainer.global_step,
),
)
# endregion
if "test" in conf.dataset:
# region Predicate classification testing callbacks
if conf.losses["bce"]["weight"] > 0:
Average(
output_transform=lambda o: o["losses"]["loss/bce"],
device=conf.session.device,
).attach(pred_class_tester, "loss/bce")
if conf.losses["rank"]["weight"] > 0:
Average(
output_transform=lambda o: o["losses"]["loss/rank"],
device=conf.session.device,
).attach(pred_class_tester, "loss/rank")
Average(
output_transform=lambda o: o["losses"]["loss/total"],
device=conf.session.device,
).attach(pred_class_tester, "loss/total")
PredicateClassificationMeanAveragePrecisionEpoch(
itemgetter("target", "output")).attach(pred_class_tester, "pc/mAP")
RecallAtEpoch((5, 10),
itemgetter("target",
"output")).attach(pred_class_tester,
"pc/recall_at")
ProgressBar(persist=True,
desc="Pred class test").attach(pred_class_tester)
pred_class_tester.add_event_handler(
Events.EPOCH_COMPLETED,
log_metrics,
"Predicate classification test",
"test",
json_logger,
tb_logger,
pred_class_trainer.global_step,
)
pred_class_tester.add_event_handler(
Events.EPOCH_COMPLETED,
PredicateClassificationLogger(
grid=(2, 3),
tag="test",
logger=tb_img_logger.writer,
metadata=dataset_metadata["test"],
global_step_fn=pred_class_trainer.global_step,
),
)
# endregion
# region Run
log_effective_config(conf, pred_class_trainer, tb_logger)
if not ("resume" in conf and conf.resume.test_only):
max_epochs = conf.session.max_epochs
if "resume" in conf:
max_epochs += pred_class_trainer.state.epoch
pred_class_trainer.run(
dataloaders["train"],
max_epochs=max_epochs,
seed=conf.session.seed,
epoch_length=len(dataloaders["train"]),
)
if "test" in conf.dataset:
pred_class_tester.run(dataloaders["test"])
add_session_end(tb_logger.writer, "SUCCESS")
tb_logger.close()
tb_img_logger.close()
import json
import pyaml
import glob
import collections
import os
os.chdir(os.path.join(os.pardir, 'data'))
for filename in glob.glob('*.json'):
jsonfile = os.path.splitext(filename)[0]
yamlfile = jsonfile + '.yaml'
with open(jsonfile + ".json", "r") as jfile:
payload = json.load(jfile, object_pairs_hook=collections.OrderedDict)
with open(
yamlfile,
'w',
) as yfile:
pyaml.dump(payload, yfile, safe=True)
print(payload)
def test_dst(self):
buff = io.BytesIO()
self.assertIs(pyaml.dump(data, buff), None)
self.assertIsInstance(pyaml.dump(data, str), str)
self.assertIsInstance(pyaml.dump(data, unicode), unicode)
def render_to_stream(self, stream):
for fname, obj in self.render():
click.secho('---\n# File: {}'.format(fname),
file=stream,
fg='blue')
pyaml.dump(obj, stream, safe=True)
def test_ids(self):
b = pyaml.dump(data, unicode)
self.assertNotIn('&id00', b)
self.assertIn('query_dump_clone: *query_dump_clone', b)
self.assertIn("'id в уникоде': &ids_-_id2_v_unikode",
b) # kinda bug - should be just "id"
def test_config_from_file(self, key, attr, value, tmpdir):
config_file = tmpdir.join("config.yaml")
with config_file.open("w") as fobj:
pyaml.dump({key: value}, fobj, safe=True, default_style='"')
config = Configuration(["--config-file", config_file.strpath])
assert getattr(config, attr) == value
def test_str_long(self):
b = pyaml.dump(data_str_long, unicode)
self.assertNotIn('"', b)
self.assertNotIn("'", b)
self.assertEqual(len(b.splitlines()), 1)
def form_submissions(hashid, format=None):
if not current_user.upgraded:
return jsonerror(402, {'error': "Please upgrade your account."})
form = Form.get_with_hashid(hashid)
for cont in form.controllers:
if cont.id == current_user.id: break
else:
if request_wants_json():
return jsonerror(403, {'error': "You do not control this form."})
else:
return redirect(url_for('dashboard'))
if not format:
# normal request.
if request_wants_json():
return jsonify({
'host': form.host,
'email': form.email,
'submissions': [dict(s.data, date=s.submitted_at.isoformat()) for s in form.submissions]
})
else:
fields = set()
for s in form.submissions:
fields.update(s.data.keys())
fields -= EXCLUDE_KEYS
submissions = []
for sub in form.submissions:
for f in fields:
value = sub.data.get(f, '')
typ = type(value)
sub.data[f] = value if typ is unicode or typ is str \
else pyaml.dump(value, safe=True)
submissions.append(sub)
return render_template('forms/submissions.html',
form=form,
fields=sorted(fields),
submissions=submissions
)
elif format:
# an export request, format can be json or csv
if format == 'json':
return Response(
json.dumps({
'host': form.host,
'email': form.email,
'submissions': [dict(s.data, date=s.submitted_at.isoformat()) for s in form.submissions]
}, sort_keys=True, indent=2),
mimetype='application/json',
headers={
'Content-Disposition': 'attachment; filename=form-%s-submissions-%s.json' \
% (hashid, datetime.datetime.now().isoformat().split('.')[0])
}
)
elif format == 'csv':
out = io.BytesIO()
fieldnames = set(field for sub in form.submissions for field in sub.data.keys())
fieldnames = ['date'] + sorted(fieldnames)
w = csv.DictWriter(out, fieldnames=fieldnames, encoding='utf-8')
w.writeheader()
for sub in form.submissions:
w.writerow(dict(sub.data, date=sub.submitted_at.isoformat()))
return Response(
out.getvalue(),
mimetype='text/csv',
headers={
'Content-Disposition': 'attachment; filename=form-%s-submissions-%s.csv' \
% (hashid, datetime.datetime.now().isoformat().split('.')[0])
}
)
def test_namedtuple(self):
TestTuple = namedtuple('TestTuple', 'y x z')
val = TestTuple(1, 2, 3)
val_str = pyaml.dump(val)
self.assertEqual(
val_str, u'y: 1\nx: 2\nz: 3\n') # namedtuple order was preserved
def main(PULL=0, VARS="test_vars.yml"):
build_config = pull_and_protein.main(PULL=PULL, )
open(VARS, 'w').write(pyaml.dump(build_config))
build_edition.main(varsfile=VARS)
def to_yaml(self):
return pyaml.dump(self.to_dict())
dn = os.path.basename(dirname)
directory[dn] = []
if dirnames:
for d in dirnames:
directory[dn].append(dir_to_dict(path=os.path.join(path, d)))
for f in filenames:
directory[dn].append(f)
else:
directory[dn] = filenames
return directory
if len(sys.argv) == 1:
p = os.getcwd()
elif len(sys.argv) == 2:
p = os.path.abspath(sys.argv[1])
else:
sys.stderr.write("Unexpected argument {}\n".format(sys.argv[2:]))
try:
with open("{}.yaml".format(os.path.basename(p)), "w") as f:
try:
yaml.dump(dir_to_dict(path=p), f, default_flow_style=False)
print("Dictionary written to {}.yaml".format(os.path.basename(p)))
except Exception as e:
print(e)
except Exception as e:
print(e)
