def update(path, dryRun=False):
try:
say("Downloading test suite data...")
shepherd = APIClient(
"https://api.csswg.org/shepherd/",
version="vnd.csswg.shepherd.v1",
ca_cert_path=certifi.where(),
)
res = shepherd.get("test_suites")
if (not res) or (406 == res.status_code):
die("This version of the test suite API is no longer supported. Please update Bikeshed."
)
return
if res.content_type not in testSuiteDataContentTypes:
die("Unrecognized test suite content-type '{0}'.",
res.content_type)
return
rawTestSuiteData = res.data
except Exception as e:
die("Couldn't download test suite data. Error was:\n{0}", str(e))
return
testSuites = dict()
for rawTestSuite in rawTestSuiteData.values():
if "specs" not in rawTestSuite:
# Looks like test-suites might not have spec data at first.
# Useless, so just drop them.
continue
testSuite = {
"vshortname": rawTestSuite["name"],
"title": rawTestSuite.get("title"),
"description": rawTestSuite.get("description"),
"status": rawTestSuite.get("status"),
"url": rawTestSuite.get("uri"),
"spec": rawTestSuite["specs"][0],
}
testSuites[testSuite["spec"]] = testSuite
if not dryRun:
try:
with open(os.path.join(path, "test-suites.json"),
"w",
encoding="utf-8") as f:
f.write(
json.dumps(testSuites,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save test-suite database to disk.\n{0}", e)
say("Success!")
def update(path, dryRun=False):
try:
say("Downloading anchor data...")
shepherd = APIClient("https://api.csswg.org/shepherd/",
version="vnd.csswg.shepherd.v1")
res = shepherd.get("specifications", anchors=True, draft=True)
# http://api.csswg.org/shepherd/spec/?spec=css-flexbox-1&anchors&draft, for manual looking
if ((not res) or (406 == res.status_code)):
die("Either this version of the anchor-data API is no longer supported, or (more likely) there was a transient network error. Try again in a little while, and/or update Bikeshed. If the error persists, please report it on GitHub."
)
return
if res.content_type not in anchorDataContentTypes:
die("Unrecognized anchor-data content-type '{0}'.",
res.contentType)
return
rawSpecData = res.data
except Exception as e:
die("Couldn't download anchor data. Error was:\n{0}", str(e))
return
specs = dict()
anchors = defaultdict(list)
headings = defaultdict(dict)
for rawSpec in rawSpecData.values():
spec = genSpec(rawSpec)
specs[spec['vshortname']] = spec
specHeadings = headings[spec['vshortname']]
def setStatus(obj, status):
obj['status'] = status
return obj
rawAnchorData = ([
setStatus(x, "snapshot")
for x in linearizeAnchorTree(rawSpec.get('anchors', []))
] + [
setStatus(x, "current")
for x in linearizeAnchorTree(rawSpec.get('draft_anchors', []))
])
for rawAnchor in rawAnchorData:
rawAnchor = fixupAnchor(rawAnchor)
linkingTexts = rawAnchor['linking_text']
if linkingTexts[0] is None:
# Happens if it had no linking text at all originally
continue
if len(linkingTexts) == 1 and linkingTexts[0].strip() == "":
# Happens if it was marked with an empty lt and Shepherd still picked it up
continue
if 'section' in rawAnchor and rawAnchor['section'] == True:
addToHeadings(rawAnchor, specHeadings, spec=spec)
if rawAnchor['type'] not in ["heading"]:
addToAnchors(rawAnchor, anchors, spec=spec)
cleanSpecHeadings(headings)
methods = extractMethodData(anchors)
fors = extractForsData(anchors)
if not dryRun:
writtenPaths = set()
try:
p = os.path.join(path, "specs.json")
writtenPaths.add(p)
with io.open(p, 'w', encoding="utf-8") as f:
f.write(
json.dumps(specs,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save spec database to disk.\n{0}", e)
return
try:
for spec, specHeadings in headings.items():
p = os.path.join(path, "headings",
"headings-{0}.json".format(spec))
writtenPaths.add(p)
with io.open(p, 'w', encoding="utf-8") as f:
f.write(
json.dumps(specHeadings,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save headings database to disk.\n{0}", e)
return
try:
writtenPaths.update(writeAnchorsFile(anchors, path))
except Exception as e:
die("Couldn't save anchor database to disk.\n{0}", e)
return
try:
p = os.path.join(path, "methods.json")
writtenPaths.add(p)
with io.open(p, 'w', encoding="utf-8") as f:
f.write(
json.dumps(methods,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save methods database to disk.\n{0}", e)
return
try:
p = os.path.join(path, "fors.json")
writtenPaths.add(p)
with io.open(p, 'w', encoding="utf-8") as f:
f.write(
json.dumps(fors,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save fors database to disk.\n{0}", e)
return
say("Success!")
return writtenPaths
def update(path, dryRun=False):
say("Downloading anchor data...")
shepherd = APIClient(
"https://api.csswg.org/shepherd/",
version="vnd.csswg.shepherd.v1",
ca_cert_path=certifi.where(),
)
rawSpecData = dataFromApi(shepherd, "specifications", draft=True)
if not rawSpecData:
return
specs = dict()
anchors = defaultdict(list)
headings = defaultdict(dict)
lastMsgTime = 0
for i, rawSpec in enumerate(rawSpecData.values(), 1):
lastMsgTime = config.doEvery(
s=5,
lastTime=lastMsgTime,
action=lambda: say(
f"Downloading data for spec {i}/{len(rawSpecData)}..."),
)
rawSpec = dataFromApi(shepherd,
"specifications",
draft=True,
anchors=True,
spec=rawSpec["name"])
spec = genSpec(rawSpec)
specs[spec["vshortname"]] = spec
specHeadings = headings[spec["vshortname"]]
def setStatus(obj, status):
obj["status"] = status
return obj
rawAnchorData = [
setStatus(x, "snapshot")
for x in linearizeAnchorTree(rawSpec.get("anchors", []))
] + [
setStatus(x, "current")
for x in linearizeAnchorTree(rawSpec.get("draft_anchors", []))
]
for rawAnchor in rawAnchorData:
rawAnchor = fixupAnchor(rawAnchor)
linkingTexts = rawAnchor["linking_text"]
if linkingTexts[0] is None:
# Happens if it had no linking text at all originally
continue
if len(linkingTexts) == 1 and linkingTexts[0].strip() == "":
# Happens if it was marked with an empty lt and Shepherd still picked it up
continue
if "section" in rawAnchor and rawAnchor["section"] is True:
addToHeadings(rawAnchor, specHeadings, spec=spec)
if rawAnchor["type"] not in ["heading"]:
addToAnchors(rawAnchor, anchors, spec=spec)
cleanSpecHeadings(headings)
methods = extractMethodData(anchors)
fors = extractForsData(anchors)
if not dryRun:
writtenPaths = set()
try:
p = os.path.join(path, "specs.json")
writtenPaths.add(p)
with open(p, "w", encoding="utf-8") as f:
f.write(
json.dumps(specs,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save spec database to disk.\n{0}", e)
return
try:
for spec, specHeadings in headings.items():
p = os.path.join(path, "headings", f"headings-{spec}.json")
writtenPaths.add(p)
with open(p, "w", encoding="utf-8") as f:
f.write(
json.dumps(specHeadings,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save headings database to disk.\n{0}", e)
return
try:
writtenPaths.update(writeAnchorsFile(anchors, path))
except Exception as e:
die("Couldn't save anchor database to disk.\n{0}", e)
return
try:
p = os.path.join(path, "methods.json")
writtenPaths.add(p)
with open(p, "w", encoding="utf-8") as f:
f.write(
json.dumps(methods,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save methods database to disk.\n{0}", e)
return
try:
p = os.path.join(path, "fors.json")
writtenPaths.add(p)
with open(p, "w", encoding="utf-8") as f:
f.write(
json.dumps(fors,
ensure_ascii=False,
indent=2,
sort_keys=True))
except Exception as e:
die("Couldn't save fors database to disk.\n{0}", e)
return
say("Success!")
return writtenPaths