Example #1
0
class Server():
	def __init__(self, configfile=None, basedir=None, host="0.0.0.0", port=5000, debug=False, allowRoot=False, logConf=None):
		self._configfile = configfile
		self._basedir = basedir
		self._host = host
		self._port = port
		self._debug = debug
		self._allowRoot = allowRoot
		self._logConf = logConf
		self._server = None

		self._logger = None

		self._lifecycle_callbacks = defaultdict(list)

		self._template_searchpaths = []

	def run(self):
		if not self._allowRoot:
			self._check_for_root()

		global app
		global babel

		global printer
		global printerProfileManager
		global fileManager
		global slicingManager
		global analysisQueue
		global userManager
		global eventManager
		global loginManager
		global pluginManager
		global appSessionManager
		global pluginLifecycleManager
		global debug

		from tornado.ioloop import IOLoop
		from tornado.web import Application, RequestHandler

		import sys

		debug = self._debug

		# first initialize the settings singleton and make sure it uses given configfile and basedir if available
		s = settings(init=True, basedir=self._basedir, configfile=self._configfile)

		# then monkey patch a bunch of stuff
		util.tornado.fix_ioloop_scheduling()
		util.flask.enable_additional_translations(additional_folders=[s.getBaseFolder("translations")])

		# setup app
		self._setup_app()

		# setup i18n
		self._setup_i18n(app)

		# then initialize logging
		self._setup_logging(self._debug, self._logConf)
		self._logger = logging.getLogger(__name__)
		def exception_logger(exc_type, exc_value, exc_tb):
			self._logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_tb))
		sys.excepthook = exception_logger
		self._logger.info("Starting OctoPrint %s" % DISPLAY_VERSION)

		# then initialize the plugin manager
		pluginManager = octoprint.plugin.plugin_manager(init=True)

		printerProfileManager = PrinterProfileManager()
		eventManager = events.eventManager()
		analysisQueue = octoprint.filemanager.analysis.AnalysisQueue()
		slicingManager = octoprint.slicing.SlicingManager(s.getBaseFolder("slicingProfiles"), printerProfileManager)
		storage_managers = dict()
		storage_managers[octoprint.filemanager.FileDestinations.LOCAL] = octoprint.filemanager.storage.LocalFileStorage(s.getBaseFolder("uploads"))
		fileManager = octoprint.filemanager.FileManager(analysisQueue, slicingManager, printerProfileManager, initial_storage_managers=storage_managers)
		printer = Printer(fileManager, analysisQueue, printerProfileManager)
		appSessionManager = util.flask.AppSessionManager()
		pluginLifecycleManager = LifecycleManager(pluginManager)

		def octoprint_plugin_inject_factory(name, implementation):
			if not isinstance(implementation, octoprint.plugin.OctoPrintPlugin):
				return None
			return dict(
				plugin_manager=pluginManager,
				printer_profile_manager=printerProfileManager,
				event_bus=eventManager,
				analysis_queue=analysisQueue,
				slicing_manager=slicingManager,
				file_manager=fileManager,
				printer=printer,
				app_session_manager=appSessionManager,
				plugin_lifecycle_manager=pluginLifecycleManager,
				data_folder=os.path.join(settings().getBaseFolder("data"), name)
			)

		def settings_plugin_inject_factory(name, implementation):
			if not isinstance(implementation, octoprint.plugin.SettingsPlugin):
				return None
			default_settings = implementation.get_settings_defaults()
			get_preprocessors, set_preprocessors = implementation.get_settings_preprocessors()
			plugin_settings = octoprint.plugin.plugin_settings(name,
			                                                   defaults=default_settings,
			                                                   get_preprocessors=get_preprocessors,
			                                                   set_preprocessors=set_preprocessors)
			return dict(settings=plugin_settings)

		def settings_plugin_config_migration(name, implementation):
			if not isinstance(implementation, octoprint.plugin.SettingsPlugin):
				return

			settings_version = implementation.get_settings_version()
			settings_migrator = implementation.on_settings_migrate

			if settings_version is not None and settings_migrator is not None:
				stored_version = implementation._settings.get_int(["_config_version"])
				if stored_version is None or stored_version < settings_version:
					settings_migrator(settings_version, stored_version)
					implementation._settings.set_int(["_config_version"], settings_version)
					implementation._settings.save()

			implementation.on_settings_initialized()

		pluginManager.implementation_inject_factories=[octoprint_plugin_inject_factory, settings_plugin_inject_factory]
		pluginManager.initialize_implementations()

		settingsPlugins = pluginManager.get_implementations(octoprint.plugin.SettingsPlugin)
		for implementation in settingsPlugins:
			try:
				settings_plugin_config_migration(implementation._identifier, implementation)
			except:
				self._logger.exception("Error while trying to migrate settings for plugin {}, ignoring it".format(implementation._identifier))

		pluginManager.implementation_post_inits=[settings_plugin_config_migration]

		pluginManager.log_all_plugins()

		# initialize file manager and register it for changes in the registered plugins
		fileManager.initialize()
		pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: fileManager.reload_plugins())

		# initialize slicing manager and register it for changes in the registered plugins
		slicingManager.initialize()
		pluginLifecycleManager.add_callback(["enabled", "disabled"], lambda name, plugin: slicingManager.reload_slicers())

		# setup jinja2
		self._setup_jinja2()
		def template_enabled(name, plugin):
			if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin):
				return
			self._register_additional_template_plugin(plugin.implementation)
		def template_disabled(name, plugin):
			if plugin.implementation is None or not isinstance(plugin.implementation, octoprint.plugin.TemplatePlugin):
				return
			self._unregister_additional_template_plugin(plugin.implementation)
		pluginLifecycleManager.add_callback("enabled", template_enabled)
		pluginLifecycleManager.add_callback("disabled", template_disabled)

		# setup assets
		self._setup_assets()

		# configure timelapse
		octoprint.timelapse.configureTimelapse()

		# setup command triggers
		events.CommandTrigger(printer)
		if self._debug:
			events.DebugEventListener()

		# setup access control
		if s.getBoolean(["accessControl", "enabled"]):
			userManagerName = s.get(["accessControl", "userManager"])
			try:
				clazz = octoprint.util.get_class(userManagerName)
				userManager = clazz()
			except AttributeError, e:
				self._logger.exception("Could not instantiate user manager %s, will run with accessControl disabled!" % userManagerName)

		app.wsgi_app = util.ReverseProxied(
			app.wsgi_app,
			s.get(["server", "reverseProxy", "prefixHeader"]),
			s.get(["server", "reverseProxy", "schemeHeader"]),
			s.get(["server", "reverseProxy", "hostHeader"]),
			s.get(["server", "reverseProxy", "prefixFallback"]),
			s.get(["server", "reverseProxy", "schemeFallback"]),
			s.get(["server", "reverseProxy", "hostFallback"])
		)

		secret_key = s.get(["server", "secretKey"])
		if not secret_key:
			import string
			from random import choice
			chars = string.ascii_lowercase + string.ascii_uppercase + string.digits
			secret_key = "".join(choice(chars) for _ in xrange(32))
			s.set(["server", "secretKey"], secret_key)
			s.save()
		app.secret_key = secret_key
		loginManager = LoginManager()
		loginManager.session_protection = "strong"
		loginManager.user_callback = load_user
		if userManager is None:
			loginManager.anonymous_user = users.DummyUser
			principals.identity_loaders.appendleft(users.dummy_identity_loader)
		loginManager.init_app(app)

		if self._host is None:
			self._host = s.get(["server", "host"])
		if self._port is None:
			self._port = s.getInt(["server", "port"])

		app.debug = self._debug

		# register API blueprint
		self._setup_blueprints()

		## Tornado initialization starts here

		ioloop = IOLoop()
		ioloop.install()

		self._router = SockJSRouter(self._create_socket_connection, "/sockjs")

		upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"]))

		server_routes = self._router.urls + [
			# various downloads
			(r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("timelapse"), as_attachment=True)),
			(r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("uploads"), as_attachment=True, path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404))),
			(r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("logs"), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator))),
			# camera snapshot
			(r"/downloads/camera/current", util.tornado.UrlForwardHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))),
			# generated webassets
			(r"/static/webassets/(.*)", util.tornado.LargeResponseHandler, dict(path=os.path.join(s.getBaseFolder("generated"), "webassets")))
		]
		for name, hook in pluginManager.get_hooks("octoprint.server.http.routes").items():
			try:
				result = hook(list(server_routes))
			except:
				self._logger.exception("There was an error while retrieving additional server routes from plugin hook {name}".format(**locals()))
			else:
				if isinstance(result, (list, tuple)):
					for entry in result:
						if not isinstance(entry, tuple) or not len(entry) == 3:
							continue
						if not isinstance(entry[0], basestring):
							continue
						if not isinstance(entry[2], dict):
							continue

						route, handler, kwargs = entry
						route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:])

						self._logger.debug("Adding additional route {route} handled by handler {handler} and with additional arguments {kwargs!r}".format(**locals()))
						server_routes.append((route, handler, kwargs))

		server_routes.append((r".*", util.tornado.UploadStorageFallbackHandler, dict(fallback=util.tornado.WsgiInputContainer(app.wsgi_app), file_prefix="octoprint-file-upload-", file_suffix=".tmp", suffixes=upload_suffixes)))

		self._tornado_app = Application(server_routes)
		max_body_sizes = [
			("POST", r"/api/files/([^/]*)", s.getInt(["server", "uploads", "maxSize"])),
			("POST", r"/api/languages", 5 * 1024 * 1024)
		]

		# allow plugins to extend allowed maximum body sizes
		for name, hook in pluginManager.get_hooks("octoprint.server.http.bodysize").items():
			try:
				result = hook(list(max_body_sizes))
			except:
				self._logger.exception("There was an error while retrieving additional upload sizes from plugin hook {name}".format(**locals()))
			else:
				if isinstance(result, (list, tuple)):
					for entry in result:
						if not isinstance(entry, tuple) or not len(entry) == 3:
							continue
						if not entry[0] in util.tornado.UploadStorageFallbackHandler.BODY_METHODS:
							continue
						if not isinstance(entry[2], int):
							continue

						method, route, size = entry
						route = r"/plugin/{name}/{route}".format(name=name, route=route if not route.startswith("/") else route[1:])

						self._logger.debug("Adding maximum body size of {size}B for {method} requests to {route})".format(**locals()))
						max_body_sizes.append((method, route, size))

		self._server = util.tornado.CustomHTTPServer(self._tornado_app, max_body_sizes=max_body_sizes, default_max_body_size=s.getInt(["server", "maxSize"]))
		self._server.listen(self._port, address=self._host)

		eventManager.fire(events.Events.STARTUP)
		if s.getBoolean(["serial", "autoconnect"]):
			(port, baudrate) = s.get(["serial", "port"]), s.getInt(["serial", "baudrate"])
			printer_profile = printerProfileManager.get_default()
			connectionOptions = get_connection_options()
			if port in connectionOptions["ports"]:
				printer.connect(port=port, baudrate=baudrate, profile=printer_profile["id"] if "id" in printer_profile else "_default")

		# start up watchdogs
		if s.getBoolean(["feature", "pollWatched"]):
			# use less performant polling observer if explicitely configured
			observer = PollingObserver()
		else:
			# use os default
			observer = Observer()
		observer.schedule(util.watchdog.GcodeWatchdogHandler(fileManager, printer), s.getBaseFolder("watched"))
		observer.start()

		# run our startup plugins
		octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
		                             "on_startup",
		                             args=(self._host, self._port))

		def call_on_startup(name, plugin):
			implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin)
			if implementation is None:
				return
			implementation.on_startup(self._host, self._port)
		pluginLifecycleManager.add_callback("enabled", call_on_startup)

		# prepare our after startup function
		def on_after_startup():
			self._logger.info("Listening on http://%s:%d" % (self._host, self._port))

			# now this is somewhat ugly, but the issue is the following: startup plugins might want to do things for
			# which they need the server to be already alive (e.g. for being able to resolve urls, such as favicons
			# or service xmls or the like). While they are working though the ioloop would block. Therefore we'll
			# create a single use thread in which to perform our after-startup-tasks, start that and hand back
			# control to the ioloop
			def work():
				octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
				                             "on_after_startup")

				def call_on_after_startup(name, plugin):
					implementation = plugin.get_implementation(octoprint.plugin.StartupPlugin)
					if implementation is None:
						return
					implementation.on_after_startup()
				pluginLifecycleManager.add_callback("enabled", call_on_after_startup)

			import threading
			threading.Thread(target=work).start()
		ioloop.add_callback(on_after_startup)

		# prepare our shutdown function
		def on_shutdown():
			# will be called on clean system exit and shutdown the watchdog observer and call the on_shutdown methods
			# on all registered ShutdownPlugins
			self._logger.info("Shutting down...")
			observer.stop()
			observer.join()
			octoprint.plugin.call_plugin(octoprint.plugin.ShutdownPlugin,
			                             "on_shutdown")
			self._logger.info("Goodbye!")
		atexit.register(on_shutdown)

		def sigterm_handler(*args, **kwargs):
			# will stop tornado on SIGTERM, making the program exit cleanly
			def shutdown_tornado():
				ioloop.stop()
			ioloop.add_callback_from_signal(shutdown_tornado)
		signal.signal(signal.SIGTERM, sigterm_handler)

		try:
			# this is the main loop - as long as tornado is running, OctoPrint is running
			ioloop.start()
		except (KeyboardInterrupt, SystemExit):
			pass
		except:
			self._logger.fatal("Now that is embarrassing... Something really really went wrong here. Please report this including the stacktrace below in OctoPrint's bugtracker. Thanks!")
			self._logger.exception("Stacktrace follows:")
Example #2
0
class Server():
    def __init__(self,
                 configfile=None,
                 basedir=None,
                 host="0.0.0.0",
                 port=5000,
                 debug=False,
                 allowRoot=False,
                 logConf=None):
        self._configfile = configfile
        self._basedir = basedir
        self._host = host
        self._port = port
        self._debug = debug
        self._allowRoot = allowRoot
        self._logConf = logConf
        self._server = None

    def run(self):
        if not self._allowRoot:
            self._checkForRoot()

        global printer
        global printerProfileManager
        global fileManager
        global slicingManager
        global analysisQueue
        global userManager
        global eventManager
        global loginManager
        global pluginManager
        global appSessionManager
        global debug

        from tornado.ioloop import IOLoop
        from tornado.web import Application

        import sys

        debug = self._debug
        import subprocess
        cmd1 = "/usr/bin/reset_user_info.sh"
        subprocess.Popen(cmd1, shell=True)
        cmd2 = "echo heartbeat > /sys/class/leds/usr1/trigger"
        subprocess.Popen(cmd2, shell=True)
        import time
        time.sleep(0.5)

        # first initialize the settings singleton and make sure it uses given configfile and basedir if available
        self._initSettings(self._configfile, self._basedir)

        # then initialize logging
        self._initLogging(self._debug, self._logConf)
        logger = logging.getLogger(__name__)

        def exception_logger(exc_type, exc_value, exc_tb):
            logger.error("Uncaught exception",
                         exc_info=(exc_type, exc_value, exc_tb))

        sys.excepthook = exception_logger
        logger.info("Starting OctoPrint %s" % DISPLAY_VERSION)

        # then initialize the plugin manager
        pluginManager = octoprint.plugin.plugin_manager(init=True)

        printerProfileManager = PrinterProfileManager()
        eventManager = events.eventManager()
        analysisQueue = octoprint.filemanager.analysis.AnalysisQueue()
        slicingManager = octoprint.slicing.SlicingManager(
            settings().getBaseFolder("slicingProfiles"), printerProfileManager)
        storage_managers = dict()
        storage_managers[
            octoprint.filemanager.FileDestinations.
            LOCAL] = octoprint.filemanager.storage.LocalFileStorage(
                settings().getBaseFolder("uploads"))
        fileManager = octoprint.filemanager.FileManager(
            analysisQueue,
            slicingManager,
            printerProfileManager,
            initial_storage_managers=storage_managers)
        printer = Printer(fileManager, analysisQueue, printerProfileManager)
        appSessionManager = util.flask.AppSessionManager()

        # configure additional template folders for jinja2
        template_plugins = pluginManager.get_implementations(
            octoprint.plugin.TemplatePlugin)
        additional_template_folders = []
        for plugin in template_plugins.values():
            folder = plugin.get_template_folder()
            if folder is not None:
                additional_template_folders.append(
                    plugin.get_template_folder())

        import jinja2
        jinja_loader = jinja2.ChoiceLoader([
            app.jinja_loader,
            jinja2.FileSystemLoader(additional_template_folders)
        ])
        app.jinja_loader = jinja_loader
        del jinja2

        # configure timelapse
        octoprint.timelapse.configureTimelapse()

        # setup command triggers
        events.CommandTrigger(printer)
        if self._debug:
            events.DebugEventListener()

        if settings().getBoolean(["accessControl", "enabled"]):
            userManagerName = settings().get(["accessControl", "userManager"])
            try:
                clazz = octoprint.util.getClass(userManagerName)
                userManager = clazz()
            except AttributeError, e:
                logger.exception(
                    "Could not instantiate user manager %s, will run with accessControl disabled!"
                    % userManagerName)

        app.wsgi_app = util.ReverseProxied(
            app.wsgi_app,
            settings().get(["server", "reverseProxy", "prefixHeader"]),
            settings().get(["server", "reverseProxy", "schemeHeader"]),
            settings().get(["server", "reverseProxy", "hostHeader"]),
            settings().get(["server", "reverseProxy", "prefixFallback"]),
            settings().get(["server", "reverseProxy", "schemeFallback"]),
            settings().get(["server", "reverseProxy", "hostFallback"]))

        secret_key = settings().get(["server", "secretKey"])
        if not secret_key:
            import string
            from random import choice
            chars = string.ascii_lowercase + string.ascii_uppercase + string.digits
            secret_key = "".join(choice(chars) for _ in xrange(32))
            settings().set(["server", "secretKey"], secret_key)
            settings().save()
        app.secret_key = secret_key
        loginManager = LoginManager()
        loginManager.session_protection = "strong"
        loginManager.user_callback = load_user
        if userManager is None:
            loginManager.anonymous_user = users.DummyUser
            principals.identity_loaders.appendleft(users.dummy_identity_loader)
        loginManager.init_app(app)

        if self._host is None:
            self._host = settings().get(["server", "host"])
        if self._port is None:
            self._port = settings().getInt(["server", "port"])

        app.debug = self._debug

        from octoprint.server.api import api
        from octoprint.server.apps import apps

        # register API blueprint
        app.register_blueprint(api, url_prefix="/api")
        app.register_blueprint(apps, url_prefix="/apps")

        # also register any blueprints defined in BlueprintPlugins
        blueprint_plugins = octoprint.plugin.plugin_manager(
        ).get_implementations(octoprint.plugin.BlueprintPlugin)
        for name, plugin in blueprint_plugins.items():
            blueprint = plugin.get_blueprint()
            if blueprint is None:
                continue

            if plugin.is_blueprint_protected():
                from octoprint.server.util import apiKeyRequestHandler, corsResponseHandler
                blueprint.before_request(apiKeyRequestHandler)
                blueprint.after_request(corsResponseHandler)

            url_prefix = "/plugin/{name}".format(name=name)
            app.register_blueprint(blueprint, url_prefix=url_prefix)
            logger.debug(
                "Registered API of plugin {name} under URL prefix {url_prefix}"
                .format(name=name, url_prefix=url_prefix))

        self._router = SockJSRouter(self._createSocketConnection, "/sockjs")

        upload_suffixes = dict(
            name=settings().get(["server", "uploads", "nameSuffix"]),
            path=settings().get(["server", "uploads", "pathSuffix"]))
        self._tornado_app = Application(
            self._router.urls +
            [(r"/downloadSD", util.tornado.LkjDownFileHandler),
             (r"/downloads/timelapse/([^/]*\.mpg)",
              util.tornado.LargeResponseHandler,
              dict(path=settings().getBaseFolder("timelapse"),
                   as_attachment=True)),
             (r"/downloads/files/local/([^/]*\.(gco|gcode|g))",
              util.tornado.LargeResponseHandler,
              dict(path=settings().getBaseFolder("uploads"),
                   as_attachment=True)),
             (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler,
              dict(path=settings().getBaseFolder("logs"),
                   as_attachment=True,
                   access_validation=util.tornado.access_validation_factory(
                       app, loginManager, util.flask.admin_validator))),
             (r"/downloads/camera/current", util.tornado.UrlForwardHandler,
              dict(url=settings().get(["webcam", "snapshot"]),
                   as_attachment=True,
                   access_validation=util.tornado.access_validation_factory(
                       app, loginManager, util.flask.user_validator))),
             (r".*", util.tornado.UploadStorageFallbackHandler,
              dict(fallback=util.tornado.WsgiInputContainer(app.wsgi_app),
                   file_prefix="octoprint-file-upload-",
                   file_suffix=".tmp",
                   suffixes=upload_suffixes))])
        max_body_sizes = [("POST", r"/api/files/([^/]*)",
                           settings().getInt(["server", "uploads",
                                              "maxSize"]))]
        self._server = util.tornado.CustomHTTPServer(
            self._tornado_app,
            max_body_sizes=max_body_sizes,
            default_max_body_size=settings().getInt(["server", "maxSize"]))
        self._server.listen(self._port, address=self._host)

        eventManager.fire(events.Events.STARTUP)
        if settings().getBoolean(["serial", "autoconnect"]):
            (port, baudrate) = settings().get(
                ["serial", "port"]), settings().getInt(["serial", "baudrate"])
            printer_profile = printerProfileManager.get_default()
            connectionOptions = getConnectionOptions()
            if port in connectionOptions["ports"]:
                printer.connect(transport_option_overrides=dict(
                    port=port, baudrate=baudrate),
                                profile=printer_profile["id"]
                                if "id" in printer_profile else "_default")

        # start up watchdogs
        observer = Observer()
        observer.schedule(
            util.watchdog.GcodeWatchdogHandler(fileManager, printer),
            settings().getBaseFolder("watched"))
        observer.start()

        ioloop = IOLoop.instance()

        # run our startup plugins
        octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
                                     "on_startup",
                                     args=(self._host, self._port))

        # prepare our after startup function
        def on_after_startup():
            logger.info("Listening on http://%s:%d" % (self._host, self._port))

            # now this is somewhat ugly, but the issue is the following: startup plugins might want to do things for
            # which they need the server to be already alive (e.g. for being able to resolve urls, such as favicons
            # or service xmls or the like). While they are working though the ioloop would block. Therefore we'll
            # create a single use thread in which to perform our after-startup-tasks, start that and hand back
            # control to the ioloop
            def work():
                octoprint.plugin.call_plugin(octoprint.plugin.StartupPlugin,
                                             "on_after_startup")

            import threading
            threading.Thread(target=work).start()

        ioloop.add_callback(on_after_startup)

        # prepare our shutdown function
        def on_shutdown():
            logger.info("Goodbye!")
            observer.stop()
            observer.join()
            octoprint.plugin.call_plugin(octoprint.plugin.ShutdownPlugin,
                                         "on_shutdown")

        atexit.register(on_shutdown)

        try:
            ioloop.start()
        except KeyboardInterrupt:
            pass
        except:
            logger.fatal(
                "Now that is embarrassing... Something really really went wrong here. Please report this including the stacktrace below in OctoPrint's bugtracker. Thanks!"
            )
            logger.exception("Stacktrace follows:")