def wipe_main():
"""Execute the kcidb-spool-wipe command-line tool"""
description = \
'kcidb-spool-wipe - Remove (old) notifications from the spool'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p',
'--project',
help='ID of the Google Cloud project containing the spool. '
'Taken from credentials by default.',
default=None,
required=False)
parser.add_argument(
'-c',
'--collection',
help='The Google Firestore path to the spool collection.',
required=True)
parser.add_argument(
'until',
metavar='UNTIL',
nargs='?',
help='An ISO-8601 timestamp specifying the newest notification to be '
'removed. The default is current time.')
args = parser.parse_args()
if args.until is None:
until = None
else:
until = dateutil.parser.isoparse(args.until)
if until.tzinfo is None:
until = until.astimezone()
Client(args.collection_path, project=args.project).wipe(until=until)
def merge_main():
"""Execute the kcidb-merge command-line tool"""
description = 'kcidb-merge - Upgrade and merge I/O data sets'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'paths',
metavar='JSON_FILE',
nargs='*',
default=[],
help='Path to a JSON file with I/O data to merge'
)
args = parser.parse_args()
merged_data = io.new()
for path in args.paths:
try:
with open(path, "r") as json_file:
data = io.schema.validate(json.load(json_file))
io.merge(merged_data, data, copy_target=False, copy_source=False)
except json.decoder.JSONDecodeError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
except jsonschema.exceptions.ValidationError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 2
json.dump(merged_data, sys.stdout, indent=4, sort_keys=True)
return 0
def describe_main():
"""Execute the kcidb-describe command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-describe - Output descriptions of report objects'
parser = misc.ArgumentParser(description=description)
parser.add_argument('obj_list_name',
metavar='LIST',
choices={n
for n in io.schema.LATEST.tree if n},
help='Name of the object list to output (%(choices)s)')
parser.add_argument('ids',
metavar='ID',
nargs='*',
default=[],
help='ID of the object to limit output to')
args = parser.parse_args()
for io_data in misc.json_load_stream_fd(sys.stdin.fileno()):
io_data = io.schema.upgrade(io.schema.validate(io_data), copy=False)
oo_data = oo.from_io(io_data)
obj_map = oo_data.get(args.obj_list_name, {})
for obj_id in args.ids or obj_map:
if obj_id in obj_map:
sys.stdout.write(obj_map[obj_id].describe())
sys.stdout.write("\x00")
sys.stdout.flush()
def describe_main():
"""Execute the kcidb-describe command-line tool"""
description = 'kcidb-describe - Output descriptions of report objects'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'obj_list_name',
metavar='LIST',
choices={n for n in io.schema.LATEST.tree if n},
help='Name of the object list to output (%(choices)s)'
)
parser.add_argument(
'ids',
metavar='ID',
nargs='*',
default=[],
help='ID of the object to limit output to'
)
args = parser.parse_args()
oo_data = oo.from_io(io.schema.upgrade(json.load(sys.stdin), copy=False))
obj_map = oo_data.get(args.obj_list_name, {})
for obj_id in args.ids or obj_map:
if obj_id in obj_map:
sys.stdout.write(obj_map[obj_id].describe())
sys.stdout.write("\x00")
return 0
def publisher_publish_main():
"""Execute the kcidb-mq-publisher-publish command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = \
'kcidb-mq-publisher-publish - ' \
'Publish with a Kernel CI report publisher, print publishing IDs'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p',
'--project',
help='ID of the Google Cloud project with the message queue',
required=True)
parser.add_argument('-t',
'--topic',
help='Name of the message queue topic to publish to',
required=True)
args = parser.parse_args()
publisher = Publisher(args.project, args.topic)
def print_publishing_id(publishing_id):
print(publishing_id, file=sys.stdout)
sys.stdout.flush()
publisher.publish_iter(
(io.schema.upgrade(io.schema.validate(data), copy=False)
for data in misc.json_load_stream_fd(sys.stdin.fileno())),
done_cb=print_publishing_id)
def submit_main():
"""Execute the kcidb-submit command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = \
'kcidb-submit - Submit Kernel CI reports, print submission IDs'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p',
'--project',
help='ID of the Google Cloud project containing the message queue',
required=True)
parser.add_argument('-t',
'--topic',
help='Name of the message queue topic to publish to',
required=True)
args = parser.parse_args()
client = Client(project_id=args.project, topic_name=args.topic)
def print_submission_id(submission_id):
print(submission_id, file=sys.stdout)
sys.stdout.flush()
client.submit_iter(
(io.schema.upgrade(io.schema.validate(data), copy=False)
for data in misc.json_load_stream_fd(sys.stdin.fileno())),
done_cb=print_submission_id)
def validate_main():
"""Execute the kcidb-tests-validate command-line tool"""
description = 'kcidb-tests-validate - Validate test catalog YAML'
parser = misc.ArgumentParser(description=description)
parser.add_argument("-u",
"--urls",
action='store_true',
help="Verify URLs in the catalog are accessible")
args = parser.parse_args()
try:
catalog = yaml.safe_load(sys.stdin)
except yaml.YAMLError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
try:
schema.validate(catalog)
except jsonschema.exceptions.ValidationError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 2
if args.urls:
try:
for test in catalog.values():
requests.head(test['home']).raise_for_status()
except requests.RequestException as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 3
return 0
def validate_main():
"""Execute the kcidb-validate command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-validate - Validate I/O JSON data'
parser = misc.ArgumentParser(description=description)
parser.parse_args()
for data in misc.json_load_stream_fd(sys.stdin.fileno()):
io.schema.validate(data)
def count_main():
"""Execute the kcidb-count command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-count - Count number of objects in I/O JSON data'
parser = misc.ArgumentParser(description=description)
parser.parse_args()
for data in misc.json_load_stream_fd(sys.stdin.fileno()):
print(io.get_obj_num(io.schema.validate(data)), file=sys.stdout)
sys.stdout.flush()
def validate_main():
"""Execute the kcidb-tests-validate command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-tests-validate - Validate test catalog YAML'
parser = misc.ArgumentParser(description=description)
parser.add_argument("-u",
"--urls",
action='store_true',
help="Verify URLs in the catalog are accessible")
args = parser.parse_args()
catalog = yaml.safe_load(sys.stdin)
schema.validate(catalog)
if args.urls:
for test in catalog.values():
requests.head(test['home']).raise_for_status()
def validate_main():
"""Execute the kcidb-validate command-line tool"""
description = 'kcidb-validate - Validate I/O JSON data'
parser = misc.ArgumentParser(description=description)
parser.parse_args()
try:
data = json.load(sys.stdin)
except json.decoder.JSONDecodeError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
try:
io.schema.validate(data)
except jsonschema.exceptions.ValidationError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 2
return 0
def publisher_cleanup_main():
"""Execute the kcidb-mq-publisher-cleanup command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = \
'kcidb-mq-publisher-cleanup - Cleanup a Kernel CI report publisher'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p',
'--project',
help='ID of the Google Cloud project with the message queue',
required=True)
parser.add_argument('-t',
'--topic',
help='Name of the message queue topic to remove',
required=True)
args = parser.parse_args()
publisher = Publisher(args.project, args.topic)
publisher.cleanup()
def notify_main():
"""Execute the kcidb-notify command-line tool"""
description = 'kcidb-notify - Generate notifications for new I/O data'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'new',
metavar='NEW_FILE',
help='Path to a JSON file with new I/O data'
)
parser.add_argument(
'base',
metavar='BASE_FILE',
nargs='?',
help='Path to a JSON file with base I/O data'
)
args = parser.parse_args()
if args.base is None:
base = io.new()
else:
try:
with open(args.base, "r") as json_file:
base = io.schema.validate(json.load(json_file))
except (json.decoder.JSONDecodeError,
jsonschema.exceptions.ValidationError) as err:
print("Failed reading base file:", file=sys.stderr)
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
try:
with open(args.new, "r") as json_file:
new = io.schema.validate(json.load(json_file))
except (json.decoder.JSONDecodeError,
jsonschema.exceptions.ValidationError) as err:
print("Failed reading new file:", file=sys.stderr)
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
for notification in subscriptions.match_new_io(base, new):
sys.stdout.write(
notification.render().as_string(policy=email.policy.SMTPUTF8)
)
sys.stdout.write("\x00")
return 0
def upgrade_main():
"""Execute the kcidb-upgrade command-line tool"""
description = 'kcidb-upgrade - Upgrade I/O JSON data to latest schema'
parser = misc.ArgumentParser(description=description)
parser.parse_args()
try:
data = json.load(sys.stdin)
except json.decoder.JSONDecodeError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 1
try:
data = io.schema.upgrade(data, copy=False)
except jsonschema.exceptions.ValidationError as err:
print(misc.format_exception_stack(err), file=sys.stderr)
return 2
json.dump(data, sys.stdout, indent=4, sort_keys=True)
return 0
def submit_main():
"""Execute the kcidb-submit command-line tool"""
description = \
'kcidb-submit - Submit Kernel CI reports'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p', '--project',
help='ID of the Google Cloud project containing the message queue',
required=True
)
parser.add_argument(
'-t', '--topic',
help='Name of the message queue topic to publish to',
required=True
)
args = parser.parse_args()
data = json.load(sys.stdin)
data = io.schema.upgrade(data, copy=False)
client = Client(project_id=args.project, topic_name=args.topic)
client.submit(data)
def subscriber_init_main():
"""Execute the kcidb-mq-subscriber-init command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = \
'kcidb-mq-subscriber-init - Initialize a Kernel CI report subscriber'
parser = misc.ArgumentParser(description=description)
parser.add_argument(
'-p',
'--project',
help='ID of the Google Cloud project with the message queue',
required=True)
parser.add_argument('-t',
'--topic',
help='Name of the subscription\'s message queue topic',
required=True)
parser.add_argument('-s',
'--subscription',
help='Name of the subscription to create',
required=True)
args = parser.parse_args()
subscriber = Subscriber(args.project, args.topic, args.subscription)
subscriber.init()
def notify_main():
"""Execute the kcidb-notify command-line tool"""
sys.excepthook = misc.log_and_print_excepthook
description = 'kcidb-notify - Generate notifications for new I/O data'
parser = misc.ArgumentParser(description=description)
parser.add_argument('base',
metavar='BASE_FILE',
nargs='?',
help='Path to a JSON file with base I/O data')
args = parser.parse_args()
base = io.new()
if args.base is not None:
try:
with open(args.base, "r") as json_file:
base_reports = [
io.schema.validate(data)
for data in misc.json_load_stream_fd(json_file.fileno())
]
base = io.merge(base,
base_reports,
copy_target=False,
copy_sources=False)
except (jq.JSONParseError,
jsonschema.exceptions.ValidationError) as err:
raise Exception("Failed reading base file") from err
try:
for new in misc.json_load_stream_fd(sys.stdin.fileno()):
new = io.schema.validate(new)
for notification in subscriptions.match_new_io(base, new):
sys.stdout.write(notification.render().as_string(
policy=email.policy.SMTPUTF8))
sys.stdout.write("\x00")
sys.stdout.flush()
base = io.merge(base, [new], copy_target=False, copy_sources=False)
except (jq.JSONParseError, jsonschema.exceptions.ValidationError) as err:
raise Exception("Failed reading new I/O data") from err
def schema_main():
"""Execute the kcidb-schema command-line tool"""
description = 'kcidb-schema - Output latest I/O JSON schema'
parser = misc.ArgumentParser(description=description)
parser.parse_args()
json.dump(io.schema.LATEST.json, sys.stdout, indent=4, sort_keys=True)
