def generate_code_from_examples():
for filename in glob.glob(os.path.join(examples_dir, "*.py")):
last_mod_time = os.path.getmtime(filename)
# Collect the names of all files created from this one. If all exist
# and are older, nothing needs to be done. Otherwise just regenerate
# all.
types = ["py", "dot", "xml", "json", "provn"]
filenames = [get_filename(filename, "examples", _i) for _i in types]
exists = [os.path.exists(_i) for _i in filenames]
if all(exists):
mtime = [os.path.getmtime(_i) > last_mod_time for _i in filenames]
if all(mtime):
continue
with io.open(filename, "rt") as fh:
code_str = BASIC_HEADER + "\n\n\n" + fh.read()
# Write Python file.
with open(get_filename(filename, "examples", "py"), "wt") as fh:
fh.write(code_str)
if "datetime(" in code_str:
code_str = "from datetime import datetime\n" + code_str
# Write dot file.
exec(
code_str +
"\n\nfrom prov import dot\n"
"dot.prov_to_dot(pr, use_labels=True).write_dot('%s')\n" %
get_filename(filename, "examples", "dot"))
# Write XML.
fname = get_filename(filename, "examples", "xml")
exec(code_str + "\n\n"
"pr.serialize('%s', format='xml')" % fname)
c14n_xml(fname)
# Write JSON.
jsonfile = get_filename(filename, "examples", "json")
exec(code_str + "\n\npr.serialize('%s', format='json')" % jsonfile)
# Read again and write in a pretty fashion.
with io.open(jsonfile, "rt") as fh:
data = json.load(fh)
with io.open(jsonfile, "wt") as fh:
json.dump(data, fh, indent=4, separators=(',', ': '),
sort_keys=True)
# Finally once more with the PROV-N serialization.
exec(code_str + "\n\n"
"with open('%s', 'wt') as fh:\n"
" fh.write(pr.get_provn())" %
get_filename(filename, "examples", "provn"))
def create_rst_representation(json_file):
with io.open(json_file, "rt") as fh:
data = json.load(fh)
node_type = os.path.basename(os.path.dirname(json_file))
# Do the attributes first.
required_attributes = [("Name", "Type", "Description")]
optional_attributes = [("Name", "Type", "Description")]
for attrib in data["attributes"]:
obj = required_attributes if attrib["required"] \
else optional_attributes
obj.append((
attrib["name"],
", ".join("``%s``" % _i for _i in attrib["types"]),
attrib["description"]))
if data["name"] == "person":
title = "Person"
elif data["name"] == "software_agent":
title = "Software Agent"
elif data["name"] == "organization":
title = "Organization"
else:
title = "%s" % (data["label"])
text = TEMPLATE.format(
title=title,
title_line="^" * len(title),
description=data["description"],
two_letter_code=data["two_letter_code"],
label=data["label"],
required_attributes=make_table(required_attributes, prefix=" "),
optional_attributes=make_table(optional_attributes, prefix=" "),
json_def_file=os.path.relpath(json_file),
name=data["name"],
node_type=node_type)
if len(optional_attributes) > 1:
example = EXAMPLE_TEMPLATE.format(
example_title="Minimal Example",
description="A concrete ``%s`` node example is illustrated "
"here as a graph, in code, and in various representations. This "
"is a minimal but valid ``%s`` node. See below for a full "
"example." % (data["name"], data["name"]),
name=data["name"],
dotfile=os.path.relpath(get_filename(
json_file, node_type, "dot", "min")),
pythonfile=os.path.relpath(get_filename(
json_file, node_type, "py", "min")),
xmlfile=os.path.relpath(get_filename(
json_file, node_type, "xml", "min")),
jsonfile=os.path.relpath(get_filename(
json_file, node_type, "json", "min")),
provnfile=os.path.relpath(get_filename(
json_file, node_type, "provn", "min")),
node_type=node_type,
ex="ex_min")
example += "\n\n\n" + EXAMPLE_TEMPLATE.format(
example_title="Full Example",
description="A concrete ``%s`` node example is illustrated "
"here as a graph, in code, and in various representations. This "
"is a full ``%s`` node containing the maximum amount of "
"information." % (data["name"], data["name"]),
name=data["name"],
dotfile=os.path.relpath(get_filename(
json_file, node_type, "dot", "max")),
pythonfile=os.path.relpath(get_filename(
json_file, node_type, "py", "max")),
xmlfile=os.path.relpath(get_filename(
json_file, node_type, "xml", "max")),
jsonfile=os.path.relpath(get_filename(
json_file, node_type, "json", "max")),
provnfile=os.path.relpath(get_filename(
json_file, node_type, "provn", "max")),
node_type=node_type,
ex="ex_max")
else:
example = EXAMPLE_TEMPLATE.format(
example_title="Example",
description="A concrete ``%s`` node example is illustrated "
"here as a graph, in code, and in various representations." %
data["name"],
name=data["name"],
dotfile=os.path.relpath(get_filename(
json_file, node_type, "dot", "max")),
pythonfile=os.path.relpath(get_filename(
json_file, node_type, "py", "max")),
xmlfile=os.path.relpath(get_filename(
json_file, node_type, "xml", "max")),
jsonfile=os.path.relpath(get_filename(
json_file, node_type, "json", "max")),
provnfile=os.path.relpath(get_filename(
json_file, node_type, "provn", "max")),
node_type=node_type,
ex="ex")
return text + "\n\n\n" + example
def generate_code():
for filename in json_files(definitions_dir):
last_mod_time = os.path.getmtime(filename)
node_type = os.path.basename(os.path.dirname(filename))
# Collect the names of all files created from this one. If all exist
# and are older, nothing needs to be done. Otherwise just regenerate
# all.
types = ["py", "dot", "xml", "json", "provn"]
filenames = [
get_filename(filename, node_type, t, i)
for t, i in itertools.product(types, ("min", "max"))]
exists = [os.path.exists(_i) for _i in filenames]
if all(exists):
mtime = [os.path.getmtime(_i) > last_mod_time for _i in filenames]
if all(mtime):
continue
with io.open(filename, "rt") as fh:
definition = json.load(fh)
# We will always generate two sets: a minimal one containing only the
# bare minimum amount of information and a maximal one containing as
# much as possible.
# Create the minimum one.
contents = []
for attrib in definition["attributes"]:
if not attrib["required"]:
continue
# For the example, only use the first type.
d_type = attrib["types"][0]
# Strings are simple.
if d_type == "xsd:string":
contents.append('("%s:%s", "%s")' % (
NS_PREFIX, attrib["name"], attrib["example_value"]))
# Others harders as they have to be represented as literals.
else:
contents.append(
'("{prefix}:{name}", prov.model.Literal(\n'
' {repr},\n prov.constants.{type}))'.format(
prefix=NS_PREFIX,
name=attrib["name"],
repr=repr(attrib["example_value"]),
type=d_type.replace(":", "_").upper()))
contents = ",\n ".join(contents)
if contents:
contents = " " + contents
if definition["type"] in ["activity", "entity"]:
name = '"%s:%s"' % (NS_PREFIX, definition["name"])
label = definition["label"]
elif definition["type"] == "agent":
if definition["name"] == "software_agent":
name = ('prov.identifier.QualifiedName(prov.constants.PROV, '
'"SoftwareAgent")')
label = [_i for _i in definition["attributes"]
if _i["name"] == "software_name"][0]["example_value"]
elif definition["name"] == "person":
name = ('prov.identifier.QualifiedName(prov.constants.PROV, '
'"Person")')
label = [_i for _i in definition["attributes"]
if _i["name"] == "name"][0]["example_value"]
elif definition["name"] == "organization":
name = ('prov.identifier.QualifiedName(prov.constants.PROV, '
'"Organization")')
label = [_i for _i in definition["attributes"]
if _i["name"] == "name"][0]["example_value"]
else:
raise ValueError
else:
raise ValueError
min_file_contents = TEMPLATE.format(
prefix=NS_PREFIX,
type=definition["type"],
two_letter_code=definition["two_letter_code"],
# Generate hash that is constant per type so the examples don't
# change each time they are regenerated.
short_hash=str(uuid.uuid5(uuid.NAMESPACE_OID, definition["type"] +
definition["two_letter_code"] +
"min_file")).replace("-", "")[:7],
label=label,
name=name,
contents=contents)
with open(get_filename(filename, node_type, "py", "min"), "wt") as fh:
fh.write(min_file_contents)
# Create the maximum one.
contents = []
for attrib in definition["attributes"]:
if "skip_example_in_doc" in attrib and \
attrib["skip_example_in_doc"]:
continue
# For the example, only use the first type.
d_type = attrib["types"][0]
# Strings are simple.
if d_type == "xsd:string":
contents.append('("%s:%s", "%s")' % (
NS_PREFIX, attrib["name"], attrib["example_value"]))
# Others harder as they have to be represented as literals.
else:
contents.append(
'("{prefix}:{name}", prov.model.Literal(\n'
' {repr},\n prov.constants.{type}))'.format(
prefix=NS_PREFIX,
name=attrib["name"],
repr=repr(attrib["example_value"]),
type=d_type.replace(":", "_").upper()))
contents = ",\n ".join(contents)
if contents:
contents = " " + contents
max_file_contents = TEMPLATE.format(
prefix=NS_PREFIX,
type=definition["type"],
two_letter_code=definition["two_letter_code"],
# Generate hash that is constant per type so the examples don't
# change each time they are regenerated.
short_hash=str(uuid.uuid5(uuid.NAMESPACE_OID, definition["type"] +
definition["two_letter_code"] +
"max_file")).replace("-", "")[:7],
label=label,
name=name,
url=NS_URL,
contents=contents)
with open(get_filename(filename, node_type, "py", "max"), "wt") as fh:
fh.write(max_file_contents)
# Create dot files with some more code generation.
exec(
min_file_contents +
"\n\nfrom prov import dot\n"
"dot.prov_to_dot(pr, use_labels=True).write_dot('%s')\n" %
get_filename(filename, node_type, "dot", "min"))
exec(
max_file_contents +
"\n\nfrom prov import dot\n"
"dot.prov_to_dot(pr, use_labels=True).write_dot('%s')\n" %
get_filename(filename, node_type, "dot", "max"))
# Same with the XML files.
fname = get_filename(filename, node_type, "xml", "min")
exec(min_file_contents + "\n\n"
"pr.serialize('%s', format='xml')" % fname)
c14n_xml(fname)
fname = get_filename(filename, node_type, "xml", "max")
exec(max_file_contents + "\n\n"
"pr.serialize('%s', format='xml')" % fname)
c14n_xml(fname)
# And once again with JSON.
jsonfile = get_filename(filename, node_type, "json", "min")
exec(min_file_contents + "\n\n"
"pr.serialize('%s', format='json')" % jsonfile)
# Read again and write in a pretty fashion.
with io.open(jsonfile, "rt") as fh:
data = json.load(fh)
with io.open(jsonfile, "wt") as fh:
json.dump(data, fh, indent=4, separators=(',', ': '),
sort_keys=True)
jsonfile = get_filename(filename, node_type, "json", "max")
exec(max_file_contents + "\n\n"
"pr.serialize('%s', format='json')" % jsonfile)
# Read again and write in a pretty fashion.
with io.open(jsonfile, "rt") as fh:
data = json.load(fh)
with io.open(jsonfile, "wt") as fh:
json.dump(data, fh, indent=4, separators=(',', ': '),
sort_keys=True)
# Finally once more with the PROV-N serialization.
exec(min_file_contents + "\n\n"
"with open('%s', 'wt') as fh:\n"
" fh.write(pr.get_provn())" %
get_filename(filename, node_type, "provn", "min"))
exec(min_file_contents + "\n\n"
"with open('%s', 'wt') as fh:\n"
" fh.write(pr.get_provn())" %
get_filename(filename, node_type, "provn", "max"))
