def accessible(self, path):
listdir = cherrypy.config.get("listdir")
showpy = cherrypy.config.get("showpy")
if os.path.isdir(path):
return listdir
elif UrlAnalyzer.is_python_file(path):
config_file = ".".join(path.split(".")[:-1]) + ".yaml"
if os.path.exists(config_file):
try:
config = tangelo.util.yaml_safe_load(config_file, dict)
except (ValueError, TypeError):
tangelo.log_warning("Config file %s could not be read - locking down associated web service source" % (path))
return False
else:
config_showpy = config.get("show-py")
if config_showpy is not None:
if not isinstance(config_showpy, bool):
tangelo.log_warning("Config file %s has a non-boolean 'show-py' property - locking down associated web service source" % (path))
return False
return config_showpy
return showpy
elif UrlAnalyzer.is_service_config_file(path):
return False
else:
return True
def unload(self, plugin_name):
tangelo.log_info("PLUGIN", "Unloading plugin '%s'" % (plugin_name))
plugin = self.plugins[plugin_name]
if plugin.module is not None:
tangelo.log_info("PLUGIN",
"\t...removing module %s" % (plugin.module))
del sys.modules[plugin.module]
for app_path in plugin.apps:
tangelo.log_info("PLUGIN",
"\t...unmounting app at %s" % (app_path))
del cherrypy.tree.apps[app_path]
if "teardown" in dir(plugin.control):
tangelo.log_info("PLUGIN", "\t...running teardown")
try:
plugin.control.teardown(
cherrypy.config["plugin-config"][plugin.path],
cherrypy.config["plugin-store"][plugin.path])
except:
tangelo.log_warning("PLUGIN", "Could not run teardown:\n%s",
(traceback.format_exc()))
del self.plugins[plugin_name]
tangelo.log_info("plugin %s unloaded" % (plugin_name))
def stream_start(url, kwargs):
content = tangelo.server.analyze_url(url).content
if content is None or content.type != Content.Service:
tangelo.http_status(500, "Error Opening Streaming Service")
return {"error": "could not open streaming service"}
else:
# Extract the path to the service and the list of positional
# arguments.
module_path = content.path
pargs = content.pargs
# Get the service module.
try:
service = modules.get(module_path)
except:
tangelo.http_status(501, "Error Importing Streaming Service")
tangelo.content_type("application/json")
return tangelo.util.traceback_report(
error="Could not import module %s" % (module_path))
else:
# Check for a "stream" function inside the module.
if "stream" not in dir(service):
tangelo.http_status(400, "Non-Streaming Service")
return {
"error":
"The requested streaming service does not implement a 'stream()' function"
}
else:
# Call the stream function and capture its result.
try:
stream = service.stream(*pargs, **kwargs)
except Exception:
result = tangelo.util.traceback_report(
error=
"Caught exception during streaming service execution",
module=tangelo.request_path())
tangelo.log_warning(
"STREAM", "Could not execute service %s:\n%s" %
(tangelo.request_path(), "\n".join(
result["traceback"])))
tangelo.http_status(500,
"Streaming Service Raised Exception")
tangelo.content_type("application/json")
return result
else:
# Generate a key corresponding to this object.
key = tangelo.util.generate_key(streams)
# Log the object in the streaming table.
streams[key] = stream
# Create an object describing the logging of the generator object.
return {"key": key}
def unload(self, plugin_name):
tangelo.log("PLUGIN", "Unloading plugin '%s'" % (plugin_name))
plugin = self.plugins[plugin_name]
if plugin.module is not None:
tangelo.log("PLUGIN", "\t...removing module %s" % (plugin.module))
del sys.modules[plugin.module]
exec("del %s" % (plugin.module))
for app_path in plugin.apps:
tangelo.log("PLUGIN", "\t...unmounting app at %s" % (app_path))
del cherrypy.tree.apps[app_path]
if "teardown" in dir(plugin.control):
tangelo.log("PLUGIN", "\t...running teardown")
try:
plugin.control.teardown(cherrypy.config["plugin-config"][plugin.path], cherrypy.config["plugin-store"][plugin.path])
except:
tangelo.log_warning("PLUGIN", "Could not run teardown:\n%s", (traceback.format_exc()))
del self.plugins[plugin_name]
def get(self, module):
# Import the module if not already imported previously (or if the module
# to import, or its configuration file, has been updated since the last
# import).
stamp = self.modules.get(module)
mtime = os.path.getmtime(module)
config_file = module[:-2] + "yaml"
config_mtime = None
if self.config:
if os.path.exists(config_file):
config_mtime = os.path.getmtime(config_file)
if (stamp is None or mtime > stamp["mtime"] or
(config_mtime is not None and config_mtime > stamp["mtime"])):
# Load any configuration the module might carry with it.
if config_mtime is not None:
try:
config = yaml_safe_load(config_file, type=dict)
except TypeError as e:
tangelo.log_warning(
"TANGELO",
"Bad configuration in file %s: %s" % (config_file, e))
raise
except IOError:
tangelo.log_warning(
"TANGELO",
"Could not open config file %s" % (config_file))
raise
except ValueError as e:
tangelo.log_warning(
"TANGELO",
"Error reading config file %s: %s" % (config_file, e))
raise
else:
config = {}
if self.config:
cherrypy.config["module-config"][module] = config
cherrypy.config["module-store"][module] = {}
# Remove .py to get the module name
name = module[:-3]
# Load the module.
service = imp.load_source(name, module)
self.modules[module] = {
"module": service,
"mtime": max(mtime, config_mtime)
}
else:
service = stamp["module"]
return service
def initialize():
global vtkpython
global weblauncher
# Get the module config.
config = tangelo.plugin_config()
# Raise an error if there's no vtkpython executable.
vtkpython = config.get("vtkpython", None)
if not vtkpython:
msg = "No 'vtkpython' option specified in configuration plugin"
tangelo.log_warning("VTKWEB", "[initialization] fatal error: %s" % (msg))
# Construct a run() function that will mask the restful API and just
# inform the caller about the configuration problem.
def run():
tangelo.http_status(400, "Bad Configuration")
return {"error": msg}
sys.modules[__name__].__dict__["run"] = run
return
vtkpython = tangelo.util.expandpath(vtkpython)
tangelo.log("VTKWEB", "[initialization] Using vtkpython executable %s" % (vtkpython))
# Use the "web launcher" included with the plugin.
weblauncher = os.path.realpath("%s/../include/vtkweb-launcher.py" % (os.path.dirname(__file__)))
# Initialize a table of VTKWeb processes.
if tangelo.plugin_store().get("processes") is None:
tangelo.plugin_store()["processes"] = {}
# Check to see if a reactor is running already.
if twisted.internet.reactor.running:
threads = [t for t in threading.enumerate() if t.name == "tangelo-vtkweb-plugin"]
if len(threads) > 0:
tangelo.log_warning(
"VTKWEB", "[initialization] A reactor started by a previous loading of this plugin is already running"
)
else:
tangelo.log_warning(
"VTKWEB", "[initialization] A reactor started by someone other than this plugin is already running"
)
else:
# Start the Twisted reactor, but in a separate thread so it doesn't
# block the CherryPy main loop. Mark the thread as "daemon" so that
# when Tangelo's main thread exits, the reactor thread will be killed
# immediately.
reactor = threading.Thread(
target=twisted.internet.reactor.run, kwargs={"installSignalHandlers": False}, name="tangelo-vtkweb-plugin"
)
reactor.daemon = True
reactor.start()
tangelo.log_info("VTKWEB", "[initialization] Starting Twisted reactor")
def initialize():
global vtkpython
global weblauncher
# Get the module config.
config = tangelo.plugin_config()
# Raise an error if there's no vtkpython executable.
vtkpython = config.get("vtkpython", None)
if not vtkpython:
msg = "No 'vtkpython' option specified in configuration plugin"
tangelo.log_warning("VTKWEB", "[initialization] fatal error: %s" % (msg))
# Construct a run() function that will mask the restful API and just
# inform the caller about the configuration problem.
def run():
tangelo.http_status(400, "Bad Configuration")
return {"error": msg}
sys.modules[__name__].__dict__["run"] = run
return
vtkpython = tangelo.util.expandpath(vtkpython)
tangelo.log("VTKWEB", "[initialization] Using vtkpython executable %s" % (vtkpython))
# Use the "web launcher" included with the plugin.
weblauncher = os.path.realpath("%s/../include/vtkweb-launcher.py" % (os.path.dirname(__file__)))
# Initialize a table of VTKWeb processes.
if tangelo.plugin_store().get("processes") is None:
tangelo.plugin_store()["processes"] = {}
# Check to see if a reactor is running already.
if twisted.internet.reactor.running:
threads = [t for t in threading.enumerate() if t.name == "tangelo-vtkweb-plugin"]
if len(threads) > 0:
tangelo.log_warning("VTKWEB", "[initialization] A reactor started by a previous loading of this plugin is already running")
else:
tangelo.log_warning("VTKWEB", "[initialization] A reactor started by someone other than this plugin is already running")
else:
# Start the Twisted reactor, but in a separate thread so it doesn't
# block the CherryPy main loop. Mark the thread as "daemon" so that
# when Tangelo's main thread exits, the reactor thread will be killed
# immediately.
reactor = threading.Thread(
target=twisted.internet.reactor.run,
kwargs={"installSignalHandlers": False},
name="tangelo-vtkweb-plugin"
)
reactor.daemon = True
reactor.start()
tangelo.log_info("VTKWEB", "[initialization] Starting Twisted reactor")
def module_cache_get(cache, module):
"""
Import a module with an optional yaml config file, but only if we haven't
imported it already.
:param cache: object which holds information on which modules and config
files have been loaded and whether config files should be
loaded.
:param module: the path of the module to load.
:returns: the loaded module.
"""
if getattr(cache, "config", False):
config_file = module[:-2] + "yaml"
if config_file not in cache.config_files and os.path.exists(
config_file):
try:
config = yaml_safe_load(config_file, type=dict)
except TypeError as e:
tangelo.log_warning(
"TANGELO",
"Bad configuration in file %s: %s" % (config_file, e))
raise
except IOError:
tangelo.log_warning(
"TANGELO", "Could not open config file %s" % (config_file))
raise
except ValueError as e:
tangelo.log_warning(
"TANGELO",
"Error reading config file %s: %s" % (config_file, e))
raise
cache.config_files[config_file] = True
else:
config = {}
cherrypy.config["module-config"][module] = config
cherrypy.config["module-store"].setdefault(module, {})
# If two threads are importing the same module nearly concurrently, we
# could load it twice unless we use the import lock.
imp.acquire_lock()
try:
if module not in cache.modules:
name = module[:-3]
# load the module.
service = imp.load_source(name, module)
cache.modules[module] = service
else:
service = cache.modules[module]
finally:
imp.release_lock()
return service
def get(self, module):
# Import the module if not already imported previously (or if the module
# to import, or its configuration file, has been updated since the last
# import).
stamp = self.modules.get(module)
mtime = os.path.getmtime(module)
config_file = module[:-2] + "yaml"
config_mtime = None
if self.config:
if os.path.exists(config_file):
config_mtime = os.path.getmtime(config_file)
if (stamp is None or
mtime > stamp["mtime"] or
(config_mtime is not None and
config_mtime > stamp["mtime"])):
# Load any configuration the module might carry with it.
if config_mtime is not None:
try:
config = load_service_config(config_file)
except TypeError as e:
tangelo.log_warning("TANGELO", "Bad configuration in file %s: %s" % (config_file, e))
raise
except IOError:
tangelo.log_warning("TANGELO", "Could not open config file %s" % (config_file))
raise
except ValueError as e:
tangelo.log_warning("TANGELO", "Error reading config file %s: %s" % (config_file, e))
raise
else:
config = {}
if self.config:
cherrypy.config["module-config"][module] = config
cherrypy.config["module-store"][module] = {}
# Remove .py to get the module name
name = module[:-3]
# Load the module.
service = imp.load_source(name, module)
self.modules[module] = {"module": service,
"mtime": max(mtime, config_mtime)}
else:
service = stamp["module"]
return service
def module_cache_get(cache, module):
"""
Import a module with an optional yaml config file, but only if we haven't
imported it already.
:param cache: object which holds information on which modules and config
files have been loaded and whether config files should be
loaded.
:param module: the path of the module to load.
:returns: the loaded module.
"""
if getattr(cache, "config", False):
config_file = module[:-2] + "yaml"
if config_file not in cache.config_files and os.path.exists(config_file):
try:
config = yaml_safe_load(config_file, type=dict)
except TypeError as e:
tangelo.log_warning("TANGELO", "Bad configuration in file %s: %s" % (config_file, e))
raise
except IOError:
tangelo.log_warning("TANGELO", "Could not open config file %s" % (config_file))
raise
except ValueError as e:
tangelo.log_warning("TANGELO", "Error reading config file %s: %s" % (config_file, e))
raise
cache.config_files[config_file] = True
else:
config = {}
cherrypy.config["module-config"][module] = config
cherrypy.config["module-store"].setdefault(module, {})
# If two threads are importing the same module nearly concurrently, we
# could load it twice unless we use the import lock.
imp.acquire_lock()
try:
if module not in cache.modules:
name = module[:-3]
# load the module.
service = imp.load_source(name, module)
cache.modules[module] = service
else:
service = cache.modules[module]
finally:
imp.release_lock()
return service
def post(*pargs, **query):
args = query.get("args", "")
timeout = float(query.get("timeout", 0))
processes = tangelo.plugin_store()["processes"]
if len(pargs) == 0:
tangelo.http_status(400, "Required Argument Missing")
return {"error": "No program path was specified"}
program_url = "/" + "/".join(pargs)
content = analyze_url(program_url).content
if content is None or content.type != Content.File:
tangelo.http_status(404, "Not Found")
return {"error": "Could not find a script at %s" % (program_url)}
elif content.path is None:
tangelo.http_status(403, "Restricted")
return {"error": "The script at %s is access-restricted"}
program = content.path
# Check the user arguments.
userargs = args.split()
if "--port" in userargs:
tangelo.http_status(400, "Illegal Argument")
return {"error": "You may not specify '--port' among the arguments passed in 'args'"}
# Obtain an available port.
port = tangelo.util.get_free_port()
# Generate a unique key.
key = tangelo.util.generate_key(processes.keys())
# Detect http vs. https
scheme = "ws"
ssl_key = cherrypy.config.get("server.ssl_private_key")
ssl_cert = cherrypy.config.get("server.ssl_certificate")
# Generate command line.
cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs
if ssl_key and ssl_cert:
scheme = "wss"
cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert])
# Launch the requested process.
tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline)))
try:
process = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except (OSError, IOError) as e:
tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process")
return {"error": e.strerror}
# Capture the new process's stdout and stderr streams in
# non-blocking readers.
stdout = tangelo.util.NonBlockingReader(process.stdout)
stderr = tangelo.util.NonBlockingReader(process.stderr)
# Read from stdout to look for the signal that the process has
# started properly.
class FactoryStarted:
pass
class Failed:
pass
class Timeout:
pass
signal = "Starting factory"
if timeout <= 0:
timeout = 10
sleeptime = 0.5
wait = 0
saved_lines = []
try:
while True:
lines = stdout.readlines()
saved_lines += lines
for line in lines:
if line == "":
# This indicates that stdout has closed without
# starting the process.
raise Failed()
elif signal in line:
# This means that the server has started.
raise FactoryStarted()
# If neither failure nor success occurred in the last block
# of lines from stdout, either time out, or try again after
# a short delay.
if wait >= timeout:
raise Timeout()
wait += sleeptime
time.sleep(sleeptime)
except Timeout:
tangelo.http_status(524, "Timeout")
return {"error": "Process startup timed out"}
except Failed:
tangelo.http_status(500)
return {"error": "Process did not start up properly",
"stdout": saved_lines,
"stderr": stderr.readlines()}
except FactoryStarted:
stdout.pushlines(saved_lines)
# Create a websocket handler path dedicated to this process.
host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host
tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp")
# Log the new process in the process table, including non-blocking
# stdout and stderr readers.
processes[key] = {"port": port,
"process": process,
"stdout": stdout,
"stderr": stderr}
# Form the websocket URL from the hostname/port used in the
# request, and the newly generated key.
url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key)
return {"key": key,
"url": url}
def refresh(self):
if not os.path.exists(self.config_file):
if self.mtime > 0:
tangelo.log_warning("PLUGIN", self.missing_msg)
self.mtime = 0
self.plugins = {}
return
mtime = os.path.getmtime(self.config_file)
if mtime <= self.mtime:
return
self.mtime = mtime
try:
config = tangelo.util.PluginConfig(self.config_file)
except IOError:
tangelo.log_warning("PLUGIN", self.missing_msg)
return
except TypeError:
tangelo.log_warning("PLUGIN", "plugin config file does not contain a top-level associative array")
return
except ValueError as e:
tangelo.log_warning("PLUGIN", "error reading plugin config file: %s" % (e))
return
seen = set()
for plugin, conf in config.plugins.iteritems():
# See whether the plugin is enabled (yes by default).
enabled = conf.get("enabled", True)
if not isinstance(enabled, bool):
tangelo.log_warning("PLUGIN", "error: setting 'enabled' in configuration for plugin '%s' must be a boolean value!" % (plugin))
continue
if enabled and plugin not in self.plugins:
# Extract the plugin path.
try:
path = os.path.join(self.config_dir, conf["path"])
except KeyError:
tangelo.log_warning("PLUGIN", "error: configuration for plugin '%s' missing required setting 'path'" % (plugin))
continue
if not self.load(plugin, path):
tangelo.log_warning("PLUGIN", "Plugin %s failed to load" % (plugin))
elif not enabled and plugin in self.plugins:
self.unload(plugin)
# Record the fact that this plugin was referenced in the plugin
# config file.
seen.add(plugin)
# All plugins that are still loaded, and yet weren't mentioned in the
# config file, should be unloaded (i.e., deleting a section from the
# plugin config file is the same as leaving it there but setting
# "enabled" to False).
for plugin in filter(lambda x: x not in seen, self.plugins):
self.unload(plugin)
def invoke_service(self, module, *pargs, **kwargs):
tangelo.content_type("text/plain")
# Save the system path (be sure to *make a copy* using the list()
# function). This will be restored to undo any modification of the path
# done by the service.
origpath = list(sys.path)
# By default, the result should be an object with error message in if
# something goes wrong; if nothing goes wrong this will be replaced
# with some other object.
result = {}
# Store the modpath in the thread-local storage (tangelo.paths() makes
# use of this per-thread data, so this is the way to get the data
# across the "module boundary" properly).
modpath = os.path.dirname(module)
cherrypy.thread_data.modulepath = modpath
cherrypy.thread_data.modulename = module
# Change the current working directory to that of the service module,
# saving the old one. This is so that the service function executes as
# though it were a Python program invoked normally, and Tangelo can
# continue running later in whatever its original CWD was.
save_cwd = os.getcwd()
os.chdir(modpath)
try:
service = self.modules.get(module)
except:
tangelo.http_status(501, "Error Importing Service")
tangelo.content_type("application/json")
result = tangelo.util.traceback_report(error="Could not import module %s" % (tangelo.request_path()))
else:
# Try to run the service - either it's in a function called
# "run()", or else it's in a REST API consisting of at least one of
# "get()", "put()", "post()", or "delete()".
#
# Collect the result in a variable - depending on its type, it will
# be transformed in some way below (by default, to JSON, but may
# also raise a cherrypy exception, log itself in a streaming table,
# etc.).
try:
if "run" in dir(service):
# Call the module's run() method, passing it the positional
# and keyword args that came into this method.
result = service.run(*pargs, **kwargs)
else:
# Reaching here means it's a REST API. Check for the
# requested method, ensure that it was marked as being part
# of the API, and call it; or give a 405 error.
method = cherrypy.request.method
restfunc = service.__dict__.get(method.lower())
if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful):
result = restfunc(*pargs, **kwargs)
else:
tangelo.http_status(405, "Method Not Allowed")
tangelo.content_type("application/json")
result = {"error": "Method '%s' is not allowed in this service" % (method)}
except:
tangelo.http_status(501, "Web Service Error")
tangelo.content_type("application/json")
result = tangelo.util.traceback_report(error="Error executing service", module=tangelo.request_path())
tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join(result["traceback"])))
# Restore the path to what it was originally.
sys.path = origpath
# Restore the CWD to what it was before the service invocation.
os.chdir(save_cwd)
# If the result is not a string, attempt to convert it to one via JSON
# serialization. This allows services to return a Python object if they
# wish, or to perform custom serialization (such as for MongoDB results,
# etc.).
if not isinstance(result, types.StringTypes):
try:
result = json.dumps(result)
except TypeError as e:
tangelo.http_status(400, "JSON Error")
tangelo.content_type("application/json")
result = {"error": "JSON type error executing service",
"message": e.message}
else:
tangelo.content_type("application/json")
return result
def load(self, plugin_name, path):
tangelo.log("PLUGIN", "Loading plugin %s (from %s)" % (plugin_name, path))
plugin = Plugins.Plugin(path)
# Check for a configuration file.
config_file = os.path.join(path, "config.yaml")
config = {}
if os.path.exists(config_file):
try:
config = tangelo.util.load_service_config(config_file)
except TypeError as e:
tangelo.log_warning("PLUGIN", "\tBad configuration in file %s: %s" % (config_file, e))
return
except IOError:
tangelo.log_warning("PLUGIN", "\tCould not open config file %s" % (config_file))
return
except ValueError as e:
tangelo.log_warning("PLUGIN", "\tError reading config file %s: %s" % (config_file, e))
return
# Install the config and an empty dict as the plugin-level
# config and store.
cherrypy.config["plugin-config"][path] = config
cherrypy.config["plugin-store"][path] = {}
# Check for a "python" directory, and place all modules found
# there in a virtual submodule of tangelo.plugin.
python = os.path.join(path, "python")
if os.path.exists(python):
tangelo.log("PLUGIN", "\t...loading python module content")
init = os.path.join(python, "__init__.py")
if not os.path.exists(init):
tangelo.log_warning("PLUGIN", "\terror: plugin '%s' includes a 'python' directory but is missing __init.py__" % (plugin))
return False
else:
module_name = "%s.%s" % (self.base_package, plugin_name)
plugin.module = module_name
old_path = sys.path
sys.path.append(python)
try:
exec('%s = sys.modules[module_name] = self.modules.get(init)' % (module_name))
except:
tangelo.log_warning("PLUGIN", "Could not import python module content:\n%s" % (traceback.format_exc()))
sys.path = old_path
return False
finally:
sys.path = old_path
# Check for any setup that needs to be done, including any apps
# that need to be mounted.
control_file = os.path.join(path, "control.py")
if os.path.exists(control_file):
tangelo.log("PLUGIN", "\t...loading plugin control module")
try:
control = self.modules.get(control_file)
plugin.control = control
except:
tangelo.log_warning("PLUGIN", "Could not import control module:\n%s" % (traceback.format_exc()))
return False
else:
if "setup" in dir(control):
tangelo.log("PLUGIN", "\t...running plugin setup")
try:
setup = control.setup(config, cherrypy.config["plugin-store"][path])
except:
tangelo.log_warning("PLUGIN", "Could not set up plugin:\n%s" % (traceback.format_exc()))
return False
else:
for app in setup.get("apps", []):
if len(app) == 2:
(app_obj, mountpoint) = app
app_config = {}
elif len(app) == 3:
(app_obj, app_config, mountpoint) = app
else:
tangelo.log_warning("PLUGIN", "\tapp mount spec has %d item%s (should be either 2 or 3)" % (len(app), "" if len(app) == 1 else "s"))
return False
app_path = os.path.join("/plugin", plugin_name, mountpoint)
if app_path in cherrypy.tree.apps:
tangelo.log_warning("PLUGIN", "\tFailed to mount application at %s (app already mounted there)" % (app_path))
return False
else:
tangelo.log("PLUGIN", "\t...mounting application at %s" % (app_path))
cherrypy.tree.mount(app_obj, app_path, app_config)
plugin.apps.append(app_path)
self.plugins[plugin_name] = plugin
return True
def invoke_service(self, module, *pargs, **kwargs):
# TODO(choudhury): This method should attempt to load the named module,
# then invoke it with the given arguments. However, if the named
# module is "config" or something similar, the method should instead
# launch a special "config" app, which lists the available app modules,
# along with docstrings or similar. It should also allow the user to
# add/delete search paths for other modules.
tangelo.content_type("text/plain")
# Save the system path (be sure to *make a copy* using the list()
# function) - it will be modified before invoking the service, and must
# be restored afterwards.
origpath = list(sys.path)
# By default, the result should be an object with error message in if
# something goes wrong; if nothing goes wrong this will be replaced
# with some other object.
result = {}
# Store the modpath in the thread-local storage (tangelo.paths() makes
# use of this per-thread data, so this is the way to get the data
# across the "module boundary" properly).
modpath = os.path.dirname(module)
cherrypy.thread_data.modulepath = modpath
cherrypy.thread_data.modulename = module
# Extend the system path with the module's home path.
sys.path.insert(0, modpath)
try:
service = self.modules.get(module)
except:
tangelo.http_status(501, "Error Importing Service")
tangelo.content_type("application/json")
result = {"error": "Could not import module %s" % (tangelo.request_path()),
"traceback": traceback.format_exc().split("\n")}
else:
# Try to run the service - either it's in a function called
# "run()", or else it's in a REST API consisting of at least one of
# "get()", "put()", "post()", or "delete()".
#
# Collect the result in a variable - depending on its type, it will
# be transformed in some way below (by default, to JSON, but may
# also raise a cherrypy exception, log itself in a streaming table,
# etc.).
try:
if 'run' in dir(service):
# Call the module's run() method, passing it the positional
# and keyword args that came into this method.
result = service.run(*pargs, **kwargs)
else:
# Reaching here means it's a REST API. Check for the
# requested method, ensure that it was marked as being part
# of the API, and call it; or give a 405 error.
method = cherrypy.request.method
restfunc = service.__dict__.get(method.lower())
if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful):
result = restfunc(*pargs, **kwargs)
else:
tangelo.http_status(405, "Method Not Allowed")
tangelo.content_type("application/json")
result = {"error": "Method '%s' is not allowed in this service" % (method)}
except:
stacktrace = traceback.format_exc()
tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), stacktrace))
tangelo.http_status(501, "Web Service Error")
tangelo.content_type("application/json")
result = {"error": "Error executing service",
"module": tangelo.request_path(),
"traceback": stacktrace.split("\n")}
# Restore the path to what it was originally.
sys.path = origpath
# If the result is not a string, attempt to convert it to one via JSON
# serialization. This allows services to return a Python object if they
# wish, or to perform custom serialization (such as for MongoDB results,
# etc.).
if not isinstance(result, types.StringTypes):
try:
result = json.dumps(result)
except TypeError as e:
tangelo.http_status(400, "JSON Error")
tangelo.content_type("application/json")
result = {"error": "JSON type error executing service",
"message": e.message}
else:
tangelo.content_type("application/json")
return result
def post(*pargs, **query):
args = query.get("args", "")
timeout = float(query.get("timeout", 0))
if len(pargs) == 0:
tangelo.http_status(400, "Required Argument Missing")
return {"error": "No program path was specified"}
program_url = "/" + "/".join(pargs)
content = analyze_url(program_url).content
if content is None or content.type != Content.File:
tangelo.http_status(404, "Not Found")
return {"error": "Could not find a script at %s" % (program_url)}
program = content.path
# Check the user arguments.
userargs = args.split()
if "--port" in userargs:
tangelo.http_status(400, "Illegal Argument")
return {"error": "You may not specify '--port' among the arguments passed in 'args'"}
# Obtain an available port.
port = tangelo.util.get_free_port()
# Generate a unique key.
key = tangelo.util.generate_key(processes.keys())
# Detect http vs. https
scheme = "ws"
ssl_key = cherrypy.config.get("server.ssl_private_key")
ssl_cert = cherrypy.config.get("server.ssl_certificate")
# Generate command line.
cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs
if ssl_key and ssl_cert:
scheme = "wss"
cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert])
# Launch the requested process.
tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline)))
try:
process = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except (OSError, IOError) as e:
tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process")
return {"error": e.strerror}
# Capture the new process's stdout and stderr streams in
# non-blocking readers.
stdout = tangelo.util.NonBlockingReader(process.stdout)
stderr = tangelo.util.NonBlockingReader(process.stderr)
# Read from stdout to look for the signal that the process has
# started properly.
class FactoryStarted:
pass
class Failed:
pass
class Timeout:
pass
signal = "Starting factory"
if timeout <= 0:
timeout = 10
sleeptime = 0.5
wait = 0
saved_lines = []
try:
while True:
lines = stdout.readlines()
saved_lines += lines
for line in lines:
if line == "":
# This indicates that stdout has closed without
# starting the process.
raise Failed()
elif signal in line:
# This means that the server has started.
raise FactoryStarted()
# If neither failure nor success occurred in the last block
# of lines from stdout, either time out, or try again after
# a short delay.
if wait >= timeout:
raise Timeout()
wait += sleeptime
time.sleep(sleeptime)
except Timeout:
tangelo.http_status(524, "Timeout")
return {"error": "Process startup timed out"}
except Failed:
tangelo.http_status(500)
return {"error": "Process did not start up properly",
"stdout": saved_lines,
"stderr": stderr.readlines()}
except FactoryStarted:
stdout.pushlines(saved_lines)
# Create a websocket handler path dedicated to this process.
host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host
tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp")
# Log the new process in the process table, including non-blocking
# stdout and stderr readers.
processes[key] = {"port": port,
"process": process,
"stdout": stdout,
"stderr": stderr}
# Form the websocket URL from the hostname/port used in the
# request, and the newly generated key.
url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key)
return {"key": key,
"url": url}
def polite(signum, frame):
tangelo.log_warning(
"TANGELO",
"Already shutting down. To force shutdown immediately, send SIGQUIT (Ctrl-\\)."
)
def polite(signum, frame):
tangelo.log_warning("TANGELO", "Already shutting down. To force shutdown immediately, send SIGQUIT (Ctrl-\\).")
def main():
p = argparse.ArgumentParser(description="Start a Tangelo server.")
p.add_argument("-c", "--config", type=str, default=None, metavar="FILE", help="specifies configuration file to use")
p.add_argument("-a", "--access-auth", action="store_const", const=True, default=None, help="enable HTTP authentication (i.e. processing of .htaccess files) (default)")
p.add_argument("-na", "--no-access-auth", action="store_const", const=True, default=None, help="disable HTTP authentication (i.e. processing of .htaccess files)")
p.add_argument("-p", "--drop-privileges", action="store_const", const=True, default=None, help="enable privilege drop when started as superuser (default)")
p.add_argument("-np", "--no-drop-privileges", action="store_const", const=True, default=None, help="disable privilege drop when started as superuser")
p.add_argument("-s", "--sessions", action="store_const", const=True, default=None, help="enable session tracking (default)")
p.add_argument("-ns", "--no-sessions", action="store_const", const=True, default=None, help="edisable session tracking")
p.add_argument("--hostname", type=str, default=None, metavar="HOSTNAME", help="overrides configured hostname on which to run Tangelo")
p.add_argument("--port", type=int, default=None, metavar="PORT", help="overrides configured port number on which to run Tangelo")
p.add_argument("-u", "--user", type=str, default=None, metavar="USERNAME", help="specifies the user to run as when root privileges are dropped")
p.add_argument("-g", "--group", type=str, default=None, metavar="GROUPNAME", help="specifies the group to run as when root privileges are dropped")
p.add_argument("-r", "--root", type=str, default=None, metavar="DIR", help="the directory from which Tangelo will serve content")
p.add_argument("--verbose", "-v", action="store_true", help="display extra information as Tangelo starts up")
p.add_argument("--version", action="store_true", help="display Tangelo version number")
p.add_argument("--key", type=str, default=None, metavar="FILE", help="the path to the SSL key. You must also specify --cert to serve content over https.")
p.add_argument("--cert", type=str, default=None, metavar="FILE", help="the path to the SSL certificate. You must also specify --key to serve content over https.")
p.add_argument("--plugin-config", type=str, default=None, metavar="PATH", help="path to plugin configuration file")
args = p.parse_args()
# If version flag is present, print the version number and exit.
if args.version:
print tangelo_version
return 0
# Make sure user didn't specify conflicting flags.
if args.access_auth and args.no_access_auth:
tangelo.log_error("ERROR", "can't specify both --access-auth (-a) and --no-access-auth (-na) together")
return 1
if args.drop_privileges and args.no_drop_privileges:
tangelo.log_error("ERROR", "can't specify both --drop-privileges (-p) and --no-drop-privileges (-np) together")
return 1
if args.no_sessions and args.sessions:
tangelo.log_error("ERROR", "can't specify both --sessions (-s) and --no-sessions (-ns) together")
sys.exit(1)
# Figure out where this is being called from - that will be useful for a
# couple of purposes.
invocation_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/..")
# Before extracting the other arguments, compute a configuration dictionary.
# If --no-config was specified, this will be the empty dictionary;
# otherwise, check the command line arguments for a config file first, then
# look for one in a sequence of other places.
config = {}
cfg_file = args.config
if cfg_file is None:
tangelo.log("TANGELO", "No configuration file specified - using command line args and defaults")
else:
cfg_file = tangelo.util.expandpath(cfg_file)
tangelo.log("TANGELO", "Using configuration file %s" % (cfg_file))
# Get a dict representing the contents of the config file.
try:
ok = False
config = Config(cfg_file)
ok = True
except (IOError, TypeError) as e:
tangelo.log_error("TANGELO", "error: %s" % (e))
except yaml.YAMLError as e:
tangelo.log_error("TANGELO", "error while parsing config file: %s" % (e))
finally:
if not ok:
return 1
# Determine whether to use access auth.
access_auth = True
if args.access_auth is None and args.no_access_auth is None:
if config.access_auth is not None:
access_auth = config.access_auth
else:
access_auth = (args.access_auth is not None) or (not args.no_access_auth)
tangelo.log("TANGELO", "Access authentication %s" % ("enabled" if access_auth else "disabled"))
# Determine whether to perform privilege drop.
drop_privileges = True
if args.drop_privileges is None and args.no_drop_privileges is None:
if config.drop_privileges is not None:
drop_privileges = config.drop_privileges
else:
drop_privileges = (args.drop_privileges is not None) or (not args.no_drop_privileges)
# Determine whether to enable sessions.
sessions = True
if args.sessions is None and args.no_sessions is None:
if config.sessions is not None:
sessions = config.sessions
else:
sessions = (args.sessions is not None) or (not args.no_sessions)
tangelo.log("TANGELO", "Sessions %s" % ("enabled" if sessions else "disabled"))
# Extract the rest of the arguments, giving priority first to command line
# arguments, then to the configuration file (if any), and finally to a
# hard-coded default value.
hostname = args.hostname or config.hostname or "localhost"
port = args.port or config.port or 8080
user = args.user or config.user or "nobody"
group = args.group or config.group or "nobody"
tangelo.log("TANGELO", "Hostname: %s" % (hostname))
tangelo.log("TANGELO", "Port: %d" % (port))
tangelo.log("TANGELO", "Privilege drop %s" % ("enabled (if necessary)" if drop_privileges else "disabled"))
if drop_privileges:
tangelo.log("TANGELO", "\tUser: %s" % (user))
tangelo.log("TANGELO", "\tGroup: %s" % (group))
# HTTPS support
#
# Grab the ssl key file.
ssl_key = args.key or config.key
if ssl_key is not None:
ssl_key = tangelo.util.expandpath(ssl_key)
# Grab the cert file.
ssl_cert = args.cert or config.cert
if ssl_cert is not None:
ssl_cert = tangelo.util.expandpath(ssl_cert)
# In order to enable HTTPS, *both* the key and cert must be specified. If
# only one or the other is specified, this is considered an error, because
# we don't want to serve what the user is considering sensitive content over
# HTTP by default.
if ssl_key is not None and ssl_cert is not None:
cherrypy.config.update({"server.ssl_module": "pyopenssl",
"server.ssl_certificate": ssl_cert,
"server.ssl_private_key": ssl_key})
tangelo.log("TANGELO", "HTTPS enabled")
tangelo.log("TANGELO", "\tSSL Cert file: %s" % (ssl_cert))
tangelo.log("TANGELO", "\tSSL Key file: %s" % (ssl_key))
elif not (ssl_key is None and ssl_cert is None):
tangelo.log_error("TANGELO", "error: SSL key or SSL cert missing")
return 1
else:
tangelo.log("TANGELO", "HTTPS disabled")
# We need a web root - use the installed example web directory as a
# fallback. This might be found in a few different places, so try them one
# by one until we find one that exists.
#
# TODO(choudhury): shouldn't we *only* check the invocation_dir option? We
# shouldn't pick up a stray web directory that happens to be found in /usr
# if we're invoking tangelo from a totally different location.
root = args.root or config.root
if root:
root = tangelo.util.expandpath(root)
else:
default_paths = map(tangelo.util.expandpath, [sys.prefix + "/share/tangelo/web",
invocation_dir + "/share/tangelo/web",
"/usr/local/share/tangelo/web"])
tangelo.log_info("TANGELO", "Looking for default web content path")
for path in default_paths:
tangelo.log_info("TANGELO", "Trying %s" % (path))
if os.path.exists(path):
root = path
break
# TODO(choudhury): by default, should we simply serve from the current
# directory? This is how SimpleHTTPServer works, for example.
if not root:
tangelo.log_error("TANGELO", "could not find default web root directory")
return 1
tangelo.log("TANGELO", "Serving content from %s" % (root))
# Compute a default plugin configuration if it was not supplied.
if args.plugin_config is None:
plugin_cfg_file = None
default_paths = map(tangelo.util.expandpath, [sys.prefix + "/share/tangelo/plugin/plugin.conf",
invocation_dir + "/share/tangelo/plugin/plugin.conf",
"/usr/local/share/tangelo/plugin/plugin.conf"])
tangelo.log_info("TANGELO", "Looking for default plugin configuration file")
for path in default_paths:
tangelo.log_info("TANGELO", "Trying %s" % (path))
if os.path.exists(path):
plugin_cfg_file = path
break
else:
plugin_cfg_file = tangelo.util.expandpath(args.plugin_config)
# Warn if plugin file doesn't exist.
if plugin_cfg_file is None:
tangelo.log_warning("TANGELO", "Could not find a default plugin configuration file")
elif not os.path.exists(plugin_cfg_file):
tangelo.log_warning("TANGELO", "Plugin configuration file %s does not exist - create it to load plugins at runtime" % (plugin_cfg_file))
else:
tangelo.log("TANGELO", "Using plugin configuration file '%s'" % (plugin_cfg_file))
# Set the web root directory.
cherrypy.config.update({"webroot": root})
# Place an empty dict to hold per-module configuration into the global
# configuration object, and one for persistent per-module storage (the
# latter can be manipulated by the service).
cherrypy.config.update({"module-config": {}})
cherrypy.config.update({"module-store": {}})
# Analogs of the module storage dicts, but for plugins.
cherrypy.config.update({"plugin-config": {}})
cherrypy.config.update({"plugin-store": {}})
# Create a plugin manager. It is marked global so that the plugins can be
# unloaded when Tangelo exits.
plugins = tangelo.server.Plugins("tangelo.plugin", plugin_cfg_file)
cherrypy.config.update({"plugins": plugins})
# Create an instance of the main handler object.
module_cache = tangelo.util.ModuleCache()
tangelo_server = tangelo.server.Tangelo(module_cache=module_cache, plugins=plugins)
rootapp = cherrypy.Application(tangelo_server, "/")
# Place an AuthUpdate handler in the Tangelo object if access authorization
# is on.
tangelo_server.auth_update = tangelo.server.AuthUpdate(app=rootapp)
# Mount the root application object.
cherrypy.tree.mount(rootapp, config={"/": {"tools.sessions.on": sessions},
"/favicon.ico": {"tools.staticfile.on": True,
"tools.staticfile.filename": sys.prefix + "/share/tangelo/tangelo.ico"}})
# Set up the global configuration.
try:
cherrypy.config.update({"environment": "production",
"log.screen": True,
"server.socket_host": hostname,
"server.socket_port": port})
except IOError as e:
tangelo.log_error("TANGELO", "problem with config file %s: %s" % (e.filename, e.strerror))
return 1
# Try to drop privileges if requested, since we've bound to whatever port
# superuser privileges were needed for already.
if drop_privileges:
# If we're on windows, don't supply any username/groupname, and just
# assume we should drop priveleges.
if platform.system() == "Windows":
tangelo.log("TANGELO", "Performing privilege drop")
cherrypy.process.plugins.DropPrivileges(cherrypy.engine).subscribe()
elif os.getuid() == 0:
tangelo.log("TANGELO", "Performing privilege drop")
# Reaching here means we're on unix, and we are the root user, so go
# ahead and drop privileges to the requested user/group.
import grp
import pwd
# On some systems, negative uids and gids are allowed. These can
# render in Python (in particular, on OS X) as very large unsigned
# values. This function first checks to see if the input value is
# already negative; if so, there's no issue and we return it
# unchanged. Otherwise, we treat the argument as a bit
# representation of a *signed* value, check the sign bit to see if
# it *should* be a negative number, and then perform the proper
# arithmetic to turn it into a signed one.
def to_signed(val):
# If we already see a negative number, just return it.
if val < 0:
return val
# Check sign bit, and subtract the unsigned range from the value
# if it is set.
return val - 0x100000000 if val & 0x80000000 else val
# Find the UID and GID for the requested user and group.
try:
mode = "user"
value = user
uid = to_signed(pwd.getpwnam(user).pw_uid)
mode = "group"
value = group
gid = to_signed(grp.getgrnam(group).gr_gid)
except KeyError:
tangelo.log_error("TANGELO", "no such %s '%s' to drop privileges to" % (mode, value))
return 1
# Set the process home directory to be the dropped-down user's.
os.environ["HOME"] = os.path.expanduser("~%s" % (user))
# Perform the actual UID/GID change.
cherrypy.process.plugins.DropPrivileges(cherrypy.engine, uid=uid, gid=gid).subscribe()
else:
tangelo.log("TANGELO", "Not performing privilege drop (because not running as superuser)")
# Set up websocket handling. Use the pass-through subclassed version of the
# plugin so we can set a priority on it that doesn't conflict with privilege
# drop.
tangelo.websocket.WebSocketLowPriorityPlugin(cherrypy.engine).subscribe()
cherrypy.tools.websocket = ws4py.server.cherrypyserver.WebSocketTool()
# Replace the stock auth_digest and auth_basic tools with ones that have
# slightly lower priority (so the AuthUpdate tool can run before them).
cherrypy.tools.auth_basic = cherrypy.Tool("before_handler", cherrypy.lib.auth_basic.basic_auth, priority=2)
cherrypy.tools.auth_digest = cherrypy.Tool("before_handler", cherrypy.lib.auth_digest.digest_auth, priority=2)
# Install signal handlers to allow for proper cleanup/shutdown.
for sig in [signal.SIGINT, signal.SIGTERM]:
signal.signal(sig, shutdown)
# Send SIGQUIT to an immediate, ungraceful shutdown instead.
if platform.system() != "Windows":
signal.signal(signal.SIGQUIT, die)
# Start the CherryPy engine.
cherrypy.engine.start()
tangelo.log_success("TANGELO", "Server is running")
cherrypy.engine.block()
