def wipe_main(): """Execute the kcidb-spool-wipe command-line tool""" description = \ 'kcidb-spool-wipe - Remove (old) notifications from the spool' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project containing the spool. ' 'Taken from credentials by default.', default=None, required=False) parser.add_argument( '-c', '--collection', help='The Google Firestore path to the spool collection.', required=True) parser.add_argument( 'until', metavar='UNTIL', nargs='?', help='An ISO-8601 timestamp specifying the newest notification to be ' 'removed. The default is current time.') args = parser.parse_args() if args.until is None: until = None else: until = dateutil.parser.isoparse(args.until) if until.tzinfo is None: until = until.astimezone() Client(args.collection_path, project=args.project).wipe(until=until)
def merge_main(): """Execute the kcidb-merge command-line tool""" description = 'kcidb-merge - Upgrade and merge I/O data sets' parser = misc.ArgumentParser(description=description) parser.add_argument( 'paths', metavar='JSON_FILE', nargs='*', default=[], help='Path to a JSON file with I/O data to merge' ) args = parser.parse_args() merged_data = io.new() for path in args.paths: try: with open(path, "r") as json_file: data = io.schema.validate(json.load(json_file)) io.merge(merged_data, data, copy_target=False, copy_source=False) except json.decoder.JSONDecodeError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 1 except jsonschema.exceptions.ValidationError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 2 json.dump(merged_data, sys.stdout, indent=4, sort_keys=True) return 0
def describe_main(): """Execute the kcidb-describe command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-describe - Output descriptions of report objects' parser = misc.ArgumentParser(description=description) parser.add_argument('obj_list_name', metavar='LIST', choices={n for n in io.schema.LATEST.tree if n}, help='Name of the object list to output (%(choices)s)') parser.add_argument('ids', metavar='ID', nargs='*', default=[], help='ID of the object to limit output to') args = parser.parse_args() for io_data in misc.json_load_stream_fd(sys.stdin.fileno()): io_data = io.schema.upgrade(io.schema.validate(io_data), copy=False) oo_data = oo.from_io(io_data) obj_map = oo_data.get(args.obj_list_name, {}) for obj_id in args.ids or obj_map: if obj_id in obj_map: sys.stdout.write(obj_map[obj_id].describe()) sys.stdout.write("\x00") sys.stdout.flush()
def describe_main(): """Execute the kcidb-describe command-line tool""" description = 'kcidb-describe - Output descriptions of report objects' parser = misc.ArgumentParser(description=description) parser.add_argument( 'obj_list_name', metavar='LIST', choices={n for n in io.schema.LATEST.tree if n}, help='Name of the object list to output (%(choices)s)' ) parser.add_argument( 'ids', metavar='ID', nargs='*', default=[], help='ID of the object to limit output to' ) args = parser.parse_args() oo_data = oo.from_io(io.schema.upgrade(json.load(sys.stdin), copy=False)) obj_map = oo_data.get(args.obj_list_name, {}) for obj_id in args.ids or obj_map: if obj_id in obj_map: sys.stdout.write(obj_map[obj_id].describe()) sys.stdout.write("\x00") return 0
def publisher_publish_main(): """Execute the kcidb-mq-publisher-publish command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-mq-publisher-publish - ' \ 'Publish with a Kernel CI report publisher, print publishing IDs' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project with the message queue', required=True) parser.add_argument('-t', '--topic', help='Name of the message queue topic to publish to', required=True) args = parser.parse_args() publisher = Publisher(args.project, args.topic) def print_publishing_id(publishing_id): print(publishing_id, file=sys.stdout) sys.stdout.flush() publisher.publish_iter( (io.schema.upgrade(io.schema.validate(data), copy=False) for data in misc.json_load_stream_fd(sys.stdin.fileno())), done_cb=print_publishing_id)
def submit_main(): """Execute the kcidb-submit command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-submit - Submit Kernel CI reports, print submission IDs' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project containing the message queue', required=True) parser.add_argument('-t', '--topic', help='Name of the message queue topic to publish to', required=True) args = parser.parse_args() client = Client(project_id=args.project, topic_name=args.topic) def print_submission_id(submission_id): print(submission_id, file=sys.stdout) sys.stdout.flush() client.submit_iter( (io.schema.upgrade(io.schema.validate(data), copy=False) for data in misc.json_load_stream_fd(sys.stdin.fileno())), done_cb=print_submission_id)
def validate_main(): """Execute the kcidb-tests-validate command-line tool""" description = 'kcidb-tests-validate - Validate test catalog YAML' parser = misc.ArgumentParser(description=description) parser.add_argument("-u", "--urls", action='store_true', help="Verify URLs in the catalog are accessible") args = parser.parse_args() try: catalog = yaml.safe_load(sys.stdin) except yaml.YAMLError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 1 try: schema.validate(catalog) except jsonschema.exceptions.ValidationError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 2 if args.urls: try: for test in catalog.values(): requests.head(test['home']).raise_for_status() except requests.RequestException as err: print(misc.format_exception_stack(err), file=sys.stderr) return 3 return 0
def validate_main(): """Execute the kcidb-validate command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-validate - Validate I/O JSON data' parser = misc.ArgumentParser(description=description) parser.parse_args() for data in misc.json_load_stream_fd(sys.stdin.fileno()): io.schema.validate(data)
def count_main(): """Execute the kcidb-count command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-count - Count number of objects in I/O JSON data' parser = misc.ArgumentParser(description=description) parser.parse_args() for data in misc.json_load_stream_fd(sys.stdin.fileno()): print(io.get_obj_num(io.schema.validate(data)), file=sys.stdout) sys.stdout.flush()
def validate_main(): """Execute the kcidb-tests-validate command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-tests-validate - Validate test catalog YAML' parser = misc.ArgumentParser(description=description) parser.add_argument("-u", "--urls", action='store_true', help="Verify URLs in the catalog are accessible") args = parser.parse_args() catalog = yaml.safe_load(sys.stdin) schema.validate(catalog) if args.urls: for test in catalog.values(): requests.head(test['home']).raise_for_status()
def validate_main(): """Execute the kcidb-validate command-line tool""" description = 'kcidb-validate - Validate I/O JSON data' parser = misc.ArgumentParser(description=description) parser.parse_args() try: data = json.load(sys.stdin) except json.decoder.JSONDecodeError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 1 try: io.schema.validate(data) except jsonschema.exceptions.ValidationError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 2 return 0
def publisher_cleanup_main(): """Execute the kcidb-mq-publisher-cleanup command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-mq-publisher-cleanup - Cleanup a Kernel CI report publisher' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project with the message queue', required=True) parser.add_argument('-t', '--topic', help='Name of the message queue topic to remove', required=True) args = parser.parse_args() publisher = Publisher(args.project, args.topic) publisher.cleanup()
def notify_main(): """Execute the kcidb-notify command-line tool""" description = 'kcidb-notify - Generate notifications for new I/O data' parser = misc.ArgumentParser(description=description) parser.add_argument( 'new', metavar='NEW_FILE', help='Path to a JSON file with new I/O data' ) parser.add_argument( 'base', metavar='BASE_FILE', nargs='?', help='Path to a JSON file with base I/O data' ) args = parser.parse_args() if args.base is None: base = io.new() else: try: with open(args.base, "r") as json_file: base = io.schema.validate(json.load(json_file)) except (json.decoder.JSONDecodeError, jsonschema.exceptions.ValidationError) as err: print("Failed reading base file:", file=sys.stderr) print(misc.format_exception_stack(err), file=sys.stderr) return 1 try: with open(args.new, "r") as json_file: new = io.schema.validate(json.load(json_file)) except (json.decoder.JSONDecodeError, jsonschema.exceptions.ValidationError) as err: print("Failed reading new file:", file=sys.stderr) print(misc.format_exception_stack(err), file=sys.stderr) return 1 for notification in subscriptions.match_new_io(base, new): sys.stdout.write( notification.render().as_string(policy=email.policy.SMTPUTF8) ) sys.stdout.write("\x00") return 0
def upgrade_main(): """Execute the kcidb-upgrade command-line tool""" description = 'kcidb-upgrade - Upgrade I/O JSON data to latest schema' parser = misc.ArgumentParser(description=description) parser.parse_args() try: data = json.load(sys.stdin) except json.decoder.JSONDecodeError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 1 try: data = io.schema.upgrade(data, copy=False) except jsonschema.exceptions.ValidationError as err: print(misc.format_exception_stack(err), file=sys.stderr) return 2 json.dump(data, sys.stdout, indent=4, sort_keys=True) return 0
def submit_main(): """Execute the kcidb-submit command-line tool""" description = \ 'kcidb-submit - Submit Kernel CI reports' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project containing the message queue', required=True ) parser.add_argument( '-t', '--topic', help='Name of the message queue topic to publish to', required=True ) args = parser.parse_args() data = json.load(sys.stdin) data = io.schema.upgrade(data, copy=False) client = Client(project_id=args.project, topic_name=args.topic) client.submit(data)
def subscriber_init_main(): """Execute the kcidb-mq-subscriber-init command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = \ 'kcidb-mq-subscriber-init - Initialize a Kernel CI report subscriber' parser = misc.ArgumentParser(description=description) parser.add_argument( '-p', '--project', help='ID of the Google Cloud project with the message queue', required=True) parser.add_argument('-t', '--topic', help='Name of the subscription\'s message queue topic', required=True) parser.add_argument('-s', '--subscription', help='Name of the subscription to create', required=True) args = parser.parse_args() subscriber = Subscriber(args.project, args.topic, args.subscription) subscriber.init()
def notify_main(): """Execute the kcidb-notify command-line tool""" sys.excepthook = misc.log_and_print_excepthook description = 'kcidb-notify - Generate notifications for new I/O data' parser = misc.ArgumentParser(description=description) parser.add_argument('base', metavar='BASE_FILE', nargs='?', help='Path to a JSON file with base I/O data') args = parser.parse_args() base = io.new() if args.base is not None: try: with open(args.base, "r") as json_file: base_reports = [ io.schema.validate(data) for data in misc.json_load_stream_fd(json_file.fileno()) ] base = io.merge(base, base_reports, copy_target=False, copy_sources=False) except (jq.JSONParseError, jsonschema.exceptions.ValidationError) as err: raise Exception("Failed reading base file") from err try: for new in misc.json_load_stream_fd(sys.stdin.fileno()): new = io.schema.validate(new) for notification in subscriptions.match_new_io(base, new): sys.stdout.write(notification.render().as_string( policy=email.policy.SMTPUTF8)) sys.stdout.write("\x00") sys.stdout.flush() base = io.merge(base, [new], copy_target=False, copy_sources=False) except (jq.JSONParseError, jsonschema.exceptions.ValidationError) as err: raise Exception("Failed reading new I/O data") from err
def schema_main(): """Execute the kcidb-schema command-line tool""" description = 'kcidb-schema - Output latest I/O JSON schema' parser = misc.ArgumentParser(description=description) parser.parse_args() json.dump(io.schema.LATEST.json, sys.stdout, indent=4, sort_keys=True)