def log(self, level, message): log_record = AirbyteLogMessage(level=level, message=message) log_message = AirbyteMessage(type="LOG", log=log_record) print(log_message.json(exclude_unset=True))
def start(self, args): # set up parent parsers parent_parser = argparse.ArgumentParser(add_help=False) main_parser = argparse.ArgumentParser() subparsers = main_parser.add_subparsers(title="commands", dest="command") # spec subparsers.add_parser("spec", help="outputs the json configuration specification", parents=[parent_parser]) # check check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", parents=[parent_parser]) required_check_parser = check_parser.add_argument_group("required named arguments") required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") # discover discover_parser = subparsers.add_parser( "discover", help="outputs a catalog describing the source's schema", parents=[parent_parser] ) required_discover_parser = discover_parser.add_argument_group("required named arguments") required_discover_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") # read read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", parents=[parent_parser]) read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file") required_read_parser = read_parser.add_argument_group("required named arguments") required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") required_read_parser.add_argument( "--catalog", type=str, required=True, help="path to the catalog used to determine which data to read" ) # parse the args parsed_args = main_parser.parse_args(args) # execute cmd = parsed_args.command if not cmd: raise Exception("No command passed") # todo: add try catch for exceptions with different exit codes with tempfile.TemporaryDirectory() as temp_dir: if cmd == "spec": message = AirbyteMessage(type=Type.SPEC, spec=self.source.spec(logger)) print(message.json(exclude_unset=True)) sys.exit(0) raw_config = self.source.read_config(parsed_args.config) config = self.source.configure(raw_config, temp_dir) if cmd == "check": check_result = self.source.check(logger, config) if check_result.status == Status.SUCCEEDED: logger.info("Check succeeded") else: logger.error("Check failed") output_message = AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result).json(exclude_unset=True) print(output_message) sys.exit(0) elif cmd == "discover": catalog = self.source.discover(logger, config) print(AirbyteMessage(type=Type.CATALOG, catalog=catalog).json(exclude_unset=True)) sys.exit(0) elif cmd == "read": catalog = self.source.read_catalog(parsed_args.catalog) state = self.source.read_state(parsed_args.state) generator = self.source.read(logger, config, catalog, state) for message in generator: print(message.json(exclude_unset=True)) sys.exit(0) else: raise Exception("Unexpected command " + cmd)