def jq_reformat(input):
"Reformat the almost-but-not-quite JSON input to remove duplicate object names to make it acceptable to the json parser"
json_string = sh.jq('-cn',
'--stream',
'def fromstream_with_dups(i):\
foreach i as $i (\
[null, null];\
\
if ($i | length) == 2 then\
if ($i[0] | length) == 0 then .\
elif $i[0][-1]|type == "string" then\
[ ( .[0] | setpath($i[0]; getpath($i[0]) + [$i[1]]) ), .[1] ]\
else [ ( .[0] | setpath($i[0]; $i[1]) ), .[1] ]\
end\
elif ($i[0] | length) == 1 then [ null, .[0] ]\
else .\
end;\
\
if ($i | length) == 1 then\
if ($i[0] | length) == 1 then .[1]\
else empty\
end\
elif ($i[0] | length) == 0 then $i[1]\
else empty\
end\
);\
fromstream_with_dups(inputs)',
_in=input,
_tty_out=False)
return str(json_string)
def rgen(self, nxt=None):
"""json --> jsons"""
import sh
while True:
indata=(yield)
if indata is None:
continue
print >>sys.stderr, 'jq:', self.args, indata[:10]
output=sh.jq(*self.args,_in=indata)
for line in output.split('\n'):
if line:
send(nxt, line)
def __init__(self, previous, new, workdir, flags=0):
self.workdir = workdir
self.previous = previous
self.new = new
self.flags = flags
try:
repo_list = sh.tr(sh.jq(sh.curl("-s", self.REPO_URL), "-M", ".[].name"),"-d", "\"").stdout.split("\n")
except sh.ErrorReturnCode:
sys.stderr.write("Failed retrieving OpenXT repository list from Github.")
raise Error
self.repos = []
for r in repo_list:
try:
self.repos.append(Repository(r,previous,new,workdir))
except:
continue
if not self.repos:
sys.stderr.write("Unable to find any repo with both references, %s and %s." % (previous, new))
raise Error
def assess_schema_descriptions(schemas_dir, schema_files, has_jq):
"""
Ensure top-level "description" and for each property.
"""
logger = logging.getLogger("lint-raml-schema")
logger.info("Assessing schema files (https://dev.folio.org/guides/describe-schema/):")
logger.info("Found %s JSON schema files under directory '%s'", len(schema_files), schemas_dir)
issues = False
props_skipped = ["id", "metadata", "resultInfo", "tags", "totalRecords"]
for schema_fn in sorted(schema_files):
schema_pn = os.path.join(schemas_dir, schema_fn)
with open(schema_pn, "r") as schema_fh:
try:
schema_data = json.load(schema_fh)
except Exception as err:
logger.error("Trouble loading %s: %s", schema_fn, err)
issues = True
continue
try:
desc = schema_data['description']
except KeyError:
logger.error('%s: Missing top-level "description".', schema_fn)
issues = True
else:
if len(desc) < 3:
logger.error('%s: The top-level "description" is too short.', schema_fn)
issues = True
try:
properties = schema_data['properties']
except KeyError:
continue
if has_jq:
logger.debug("Doing jq")
# Use jq to gather all properties into easier-to-use form.
jq_filter = '[ .. | .properties? | objects ]'
try:
result_jq = sh.jq('--monochrome-output', jq_filter, schema_pn).stdout.decode().strip()
except sh.ErrorReturnCode_2 as err:
logger.error("Trouble doing jq: usage error: %s", err.stderr.decode())
issues = True
except sh.ErrorReturnCode_3 as err:
logger.error("Trouble doing jq: compile error: %s", err.stderr.decode())
issues = True
else:
try:
jq = json.loads(result_jq)
except Exception as err:
logger.error("Trouble loading JSON obtained from jq: %s", err)
issues = True
continue
else:
# logger.debug("JQ: %s", jq)
desc_missing = []
for props in jq:
for prop in props:
if prop in props_skipped:
continue
try:
desc = props[prop]['description']
except KeyError:
desc_missing.append(prop)
except TypeError:
logger.error('%s: Trouble determining "description" for property, perhaps misplaced.', schema_fn)
desc_missing.append("misplaced")
else:
if len(desc) < 3:
desc_missing.append(prop)
if desc_missing:
logger.error('%s: Missing "description" for: %s', schema_fn, ', '.join(sorted(desc_missing)))
issues = True
else:
logger.debug('%s: Each property "description" is present.', schema_fn)
else:
logger.warning("No 'jq' so not assessing schema file.")
return issues
def assess_schema_descriptions(schema_files):
"""
Ensure top-level "description" and for each property.
"""
logger = logging.getLogger("api-schema-lint")
issues = False
version_schema_re = re.compile(r"json-schema.org/(.+)schema#?")
props_skipped = ["id", "metadata", "resultInfo", "tags", "totalRecords"]
for schema_fn in sorted(schema_files):
schema_pn = os.path.relpath(schema_fn)
logger.debug("Processing file: %s", schema_pn)
with open(schema_pn, "r") as schema_fh:
try:
schema_data = json.load(schema_fh)
except Exception as err:
logger.error("Trouble loading %s: %s", schema_pn, err)
issues = True
continue
''' 20210417: disable until OAS 3.1 FOLIO-2948
try:
keyword_schema = schema_data['$schema']
except KeyError:
logger.warning('%s: Missing "$schema" keyword.', schema_pn)
# FIXME: Should this be an error? Best practice says yes.
#issues = True
else:
match = re.search(version_schema_re, keyword_schema)
if not match:
msg = "%s: Malformed $schema keyword: %s"
logger.error(msg, schema_pn, keyword_schema)
'''
try:
desc = schema_data['description']
except KeyError:
logger.error('%s: Missing top-level "description".', schema_pn)
issues = True
else:
if len(desc) < 3:
logger.error('%s: The top-level "description" is too short.',
schema_pn)
issues = True
try:
properties = schema_data['properties']
except KeyError:
logger.debug('%s: Has no object properties.', schema_pn)
continue
# Use jq to gather all properties into easier-to-use form.
jq_filter = '[ .. | .properties? | objects ]'
try:
result_jq = sh.jq('--monochrome-output', jq_filter,
schema_pn).stdout.decode().strip()
except sh.ErrorReturnCode_2 as err:
logger.error("Trouble doing jq: usage error: %s",
err.stderr.decode())
issues = True
except sh.ErrorReturnCode_3 as err:
logger.error("Trouble doing jq: compile error: %s",
err.stderr.decode())
issues = True
else:
try:
jq_content = json.loads(result_jq)
except Exception as err:
logger.error("Trouble loading JSON obtained from jq: %s", err)
issues = True
continue
else:
# logger.debug("JQ: %s", jq_content)
desc_missing = []
for props in jq_content:
for prop in props:
if prop in props_skipped:
continue
try:
desc = props[prop]['description']
except KeyError:
desc_missing.append(prop)
except TypeError:
msg = '%s: Trouble determining "description" for property, perhaps misplaced.'
logger.error(msg, schema_pn)
desc_missing.append("misplaced")
else:
if len(desc) < 3:
desc_missing.append(prop)
if desc_missing:
msg = '%s: Missing "description" for: %s'
logger.error(msg, schema_pn,
', '.join(sorted(desc_missing)))
issues = True
return issues
for _block in block.children:
groups = re.search('^(.*?)\:(.*?)(\|(.*))?$',
_block.title).groups()
if (groups[0] == "Category"):
dict_to_push['category'] = groups[1].strip()
if (groups[0] == "Status"):
dict_to_push['status'] = groups[1].strip()
if groups[2]:
dict_to_push['status_comment'] = groups[3].strip()
arr.append(dict_to_push)
except:
print(block)
raise ("f**k you")
with open("root.html", "r") as f:
html = f.read()
tmpl = Template(html)
with open("index.html", "w") as f:
f.write(tmpl.render(items=arr, last_modified=ts.strftime("%b %d, %Y")))
sh.jq(sh.purgecss(["--css", "assets/tailwind.css", "--content", "index.html"]),
"-r",
".[].css",
_out="assets/build.css")