def stream_start(url, kwargs): content = tangelo.server.analyze_url(url).content if content is None or content.type != Content.Service: tangelo.http_status(500, "Error Opening Streaming Service") return {"error": "could not open streaming service"} else: # Extract the path to the service and the list of positional # arguments. module_path = content.path pargs = content.pargs # Get the service module. try: service = modules.get(module_path) except: tangelo.http_status(501, "Error Importing Streaming Service") tangelo.content_type("application/json") return tangelo.util.traceback_report( error="Could not import module %s" % (module_path)) else: # Check for a "stream" function inside the module. if "stream" not in dir(service): tangelo.http_status(400, "Non-Streaming Service") return { "error": "The requested streaming service does not implement a 'stream()' function" } else: # Call the stream function and capture its result. try: stream = service.stream(*pargs, **kwargs) except Exception: result = tangelo.util.traceback_report( error= "Caught exception during streaming service execution", module=tangelo.request_path()) tangelo.log_warning( "STREAM", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join( result["traceback"]))) tangelo.http_status(500, "Streaming Service Raised Exception") tangelo.content_type("application/json") return result else: # Generate a key corresponding to this object. key = tangelo.util.generate_key(streams) # Log the object in the streaming table. streams[key] = stream # Create an object describing the logging of the generator object. return {"key": key}
def invoke_service(self, module, *pargs, **kwargs): # TODO(choudhury): This method should attempt to load the named module, # then invoke it with the given arguments. However, if the named # module is "config" or something similar, the method should instead # launch a special "config" app, which lists the available app modules, # along with docstrings or similar. It should also allow the user to # add/delete search paths for other modules. tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function) - it will be modified before invoking the service, and must # be restored afterwards. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Extend the system path with the module's home path. sys.path.insert(0, modpath) try: service = self.modules.get(module) except: tangelo.http_status(501, "Error Importing Service") tangelo.content_type("application/json") result = {"error": "Could not import module %s" % (tangelo.request_path()), "traceback": traceback.format_exc().split("\n")} else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if 'run' in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__.get(method.lower()) if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: tangelo.http_status(405, "Method Not Allowed") tangelo.content_type("application/json") result = {"error": "Method '%s' is not allowed in this service" % (method)} except: stacktrace = traceback.format_exc() tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), stacktrace)) tangelo.http_status(501, "Web Service Error") tangelo.content_type("application/json") result = {"error": "Error executing service", "module": tangelo.request_path(), "traceback": stacktrace.split("\n")} # Restore the path to what it was originally. sys.path = origpath # If the result is not a string, attempt to convert it to one via JSON # serialization. This allows services to return a Python object if they # wish, or to perform custom serialization (such as for MongoDB results, # etc.). if not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: tangelo.http_status(400, "JSON Error") tangelo.content_type("application/json") result = {"error": "JSON type error executing service", "message": e.message} else: tangelo.content_type("application/json") return result
def invoke_service(self, module, *pargs, **kwargs): # TODO(choudhury): This method should attempt to load the named module, # then invoke it with the given arguments. However, if the named # module is "config" or something similar, the method should instead # launch a special "config" app, which lists the available app modules, # along with docstrings or similar. It should also allow the user to # add/delete search paths for other modules. tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function) - it will be modified before invoking the service, and must # be restored afterwards. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Extend the system path with the module's home path. sys.path.insert(0, modpath) # Import the module if not already imported previously (or if the # module to import, or its configuration file, has been updated since # the last import). try: stamp = self.modules.get(module) mtime = os.path.getmtime(module) config_file = module[:-2] + "json" config_mtime = None if os.path.exists(config_file): config_mtime = os.path.getmtime(config_file) if (stamp is None or mtime > stamp["mtime"] or (config_mtime is not None and config_mtime > stamp["mtime"])): if stamp is None: tangelo.log("loading new module: " + module) else: tangelo.log("reloading module: " + module) # Load any configuration the module might carry with it. if config_mtime is not None: try: with open(config_file) as f: config = json.loads(json_minify(f.read())) if type(config) != dict: msg = ("Service module configuration file " + "does not contain a key-value store " + "(i.e., a JSON Object)") tangelo.log(msg) raise TypeError(msg) except IOError: tangelo.log("Could not open config file %s" % (config_file)) raise except ValueError as e: tangelo.log("Error reading config file %s: %s" % (config_file, e)) raise else: config = {} cherrypy.config["module-config"][module] = config # Remove .py to get the module name name = module[:-3] # Load the module. service = imp.load_source(name, module) self.modules[module] = { "module": service, "mtime": max(mtime, config_mtime) } else: service = stamp["module"] except: bt = traceback.format_exc() tangelo.log("Error importing module %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode( "501 Error in Python Service", Tangelo.literal + "There was an error while " + "trying to import module " + "%s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if 'run' in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__[method.lower()] if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: result = tangelo.HTTPStatusCode( 405, "Method not allowed") except Exception as e: bt = traceback.format_exc() tangelo.log( "Caught exception while executing service %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode( "501 Error in Python Service", Tangelo.literal + "There was an error " + "executing service " + "%s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) # Restore the path to what it was originally. sys.path = origpath # Check the type of the result to decide what result to finally return: # # 1. If it is an HTTPStatusCode object, raise a cherrypy HTTPError # exception, which will cause the browser to do the right thing. # # 2. TODO: If it's a Python generator object, log it with the Tangelo # streaming API. # # 3. If it's a Python dictionary, convert it to JSON. # # 4. If it's a string, don't do anything to it. # # This allows the services to return a Python object if they wish, or # to perform custom serialization (such as for MongoDB results, etc.). if isinstance(result, tangelo.HTTPStatusCode): if result.msg: raise cherrypy.HTTPError(result.code, result.msg) else: raise cherrypy.HTTPError(result.code) elif "next" in dir(result): if self.stream: return self.stream.add(result) else: return json.dumps({ "error": "Streaming is not supported " + "in this instance of Tangelo" }) elif not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: msg = Tangelo.literal + "<p>A JSON type error occurred in service " + tangelo.request_path( ) + ":</p>" msg += "<p><pre>" + cgi.escape(e.message) + "</pre></p>" raise cherrypy.HTTPError("501 Error in Python Service", msg) return result
def stream_start(url, kwargs): content = tangelo.server.analyze_url(url).content if content is None or content.type != Content.Service: tangelo.http_status(500, "Error Opening Streaming Service") return {"error": "could not open streaming service"} else: # Extract the path to the service and the list of positional # arguments. module_path = content.path pargs = content.pargs # Get the service module. try: service = modules.get(module_path) except: tangelo.http_status(500, "Error Importing Streaming Service") tangelo.content_type("application/json") error_code = tangelo.util.generate_error_code() tangelo.util.log_traceback("STREAM", error_code, "Could not import module %s" % (tangelo.request_path())) return tangelo.util.error_report(error_code) else: # Check for a "stream" function inside the module. if "stream" not in dir(service): tangelo.http_status(400, "Non-Streaming Service") return {"error": "The requested streaming service does not implement a 'stream()' function"} else: # Call the stream function and capture its result. try: stream = service.stream(*pargs, **kwargs) except Exception: tangelo.http_status(500, "Streaming Service Raised Exception") tangelo.content_type("application/json") error_code = tangelo.util.generate_error_code() tangelo.util.log_traceback("STREAM", error_code, "Could not execute service %s" % (tangelo.request_path())) return tangelo.util.error_report(error_code) else: # Generate a key corresponding to this object. key = tangelo.util.generate_key(streams) # Log the object in the streaming table. streams[key] = stream # Create an object describing the logging of the generator object. return {"key": key}
def invoke_service(self, module, *pargs, **kwargs): tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function). This will be restored to undo any modification of the path # done by the service. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Change the current working directory to that of the service module, # saving the old one. This is so that the service function executes as # though it were a Python program invoked normally, and Tangelo can # continue running later in whatever its original CWD was. save_cwd = os.getcwd() os.chdir(modpath) try: service = self.modules.get(module) except: tangelo.http_status(501, "Error Importing Service") tangelo.content_type("application/json") result = tangelo.util.traceback_report(error="Could not import module %s" % (tangelo.request_path())) else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if "run" in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__.get(method.lower()) if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: tangelo.http_status(405, "Method Not Allowed") tangelo.content_type("application/json") result = {"error": "Method '%s' is not allowed in this service" % (method)} except: tangelo.http_status(501, "Web Service Error") tangelo.content_type("application/json") result = tangelo.util.traceback_report(error="Error executing service", module=tangelo.request_path()) tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join(result["traceback"]))) # Restore the path to what it was originally. sys.path = origpath # Restore the CWD to what it was before the service invocation. os.chdir(save_cwd) # If the result is not a string, attempt to convert it to one via JSON # serialization. This allows services to return a Python object if they # wish, or to perform custom serialization (such as for MongoDB results, # etc.). if not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: tangelo.http_status(400, "JSON Error") tangelo.content_type("application/json") result = {"error": "JSON type error executing service", "message": e.message} else: tangelo.content_type("application/json") return result
def invoke_service(self, module, *pargs, **kwargs): # TODO(choudhury): This method should attempt to load the named module, # then invoke it with the given arguments. However, if the named # module is "config" or something similar, the method should instead # launch a special "config" app, which lists the available app modules, # along with docstrings or similar. It should also allow the user to # add/delete search paths for other modules. tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function) - it will be modified before invoking the service, and must # be restored afterwards. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Extend the system path with the module's home path. sys.path.insert(0, modpath) # Import the module if not already imported previously (or if the # module to import, or its configuration file, has been updated since # the last import). try: stamp = self.modules.get(module) mtime = os.path.getmtime(module) config_file = module[:-2] + "json" config_mtime = None if os.path.exists(config_file): config_mtime = os.path.getmtime(config_file) if (stamp is None or mtime > stamp["mtime"] or (config_mtime is not None and config_mtime > stamp["mtime"])): if stamp is None: tangelo.log("loading new module: " + module) else: tangelo.log("reloading module: " + module) # Load any configuration the module might carry with it. if config_mtime is not None: try: with open(config_file) as f: config = json.loads(json_minify(f.read())) if type(config) != dict: msg = ("Service module configuration file " + "does not contain a key-value store " + "(i.e., a JSON Object)") tangelo.log(msg) raise TypeError(msg) except IOError: tangelo.log("Could not open config file %s" % (config_file)) raise except ValueError as e: tangelo.log("Error reading config file %s: %s" % (config_file, e)) raise else: config = {} cherrypy.config["module-config"][module] = config # Remove .py to get the module name name = module[:-3] # Load the module. service = imp.load_source(name, module) self.modules[module] = {"module": service, "mtime": max(mtime, config_mtime)} else: service = stamp["module"] except: bt = traceback.format_exc() tangelo.log("Error importing module %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode("501 Error in Python Service", Tangelo.literal + "There was an error while " + "trying to import module " + "%s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if 'run' in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__[method.lower()] if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: result = tangelo.HTTPStatusCode(405, "Method not allowed") except Exception as e: bt = traceback.format_exc() tangelo.log("Caught exception while executing service %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode("501 Error in Python Service", Tangelo.literal + "There was an error " + "executing service " + "%s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) # Restore the path to what it was originally. sys.path = origpath # Check the type of the result to decide what result to finally return: # # 1. If it is an HTTPStatusCode object, raise a cherrypy HTTPError # exception, which will cause the browser to do the right thing. # # 2. TODO: If it's a Python generator object, log it with the Tangelo # streaming API. # # 3. If it's a Python dictionary, convert it to JSON. # # 4. If it's a string, don't do anything to it. # # This allows the services to return a Python object if they wish, or # to perform custom serialization (such as for MongoDB results, etc.). if isinstance(result, tangelo.HTTPStatusCode): if result.msg: raise cherrypy.HTTPError(result.code, result.msg) else: raise cherrypy.HTTPError(result.code) elif "next" in dir(result): if self.stream: return self.stream.add(result) else: return json.dumps({"error": "Streaming is not supported " + "in this instance of Tangelo"}) elif not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: msg = Tangelo.literal + "<p>A JSON type error occurred in service " + tangelo.request_path() + ":</p>" msg += "<p><pre>" + cgi.escape(e.message) + "</pre></p>" raise cherrypy.HTTPError("501 Error in Python Service", msg) return result
def invoke_service(self, module, *pargs, **kwargs): # TODO(choudhury): This method should attempt to load the named module, # then invoke it with the given arguments. However, if the named module # is "config" or something similar, the method should instead launch a # special "config" app, which lists the available app modules, along # with docstrings or similar. It should also allow the user to # add/delete search paths for other modules. tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function) - it will be modified before invoking the service, and must # be restored afterwards. origpath = list(sys.path) # By default, the result should be a bare response that we will place an # error message in if something goes wrong; if nothing goes wrong this # will be replaced with some other object. result = tangelo.empty_response() # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data across # the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Extend the system path with the module's home path. sys.path.insert(0, modpath) # Import the module if not already imported previously (or if the module # to import, or its configuration file, has been updated since the last # import). try: stamp = self.modules.get(module) mtime = os.path.getmtime(module) config_file = module[:-2] + "json" config_mtime = None if os.path.exists(config_file): config_mtime = os.path.getmtime(config_file) if stamp is None or mtime > stamp["mtime"] or (config_mtime is not None and config_mtime > stamp["mtime"]): if stamp is None: tangelo.log("loading new module: " + module) else: tangelo.log("reloading module: " + module) # Load any configuration the module might carry with it. if config_mtime is not None: try: with open(config_file) as f: config = json.loads(json_minify(f.read())) if type(config) != dict: msg = "Service module configuration file does not contain a key-value store (i.e., a JSON Object)" tangelo.log(msg) raise TypeError(msg) except IOError: tangelo.log("Could not open config file %s" % (config_file)) raise except ValueError as e: tangelo.log("Error reading config file %s: %s" % (config_file, e)) raise else: config = {} cherrypy.config["module-config"][module] = config # Remove .py to get the module name name = module[:-3] # Load the module. service = imp.load_source(name, module) self.modules[module] = { "module": service, "mtime": max(mtime, config_mtime) } else: service = stamp["module"] except: bt = traceback.format_exc() tangelo.log("Error importing module %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode("501 Error in Python Service", "There was an error while trying to import module %s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) else: # Try to run the service - either it's in a function called "run()", # or else it's in a REST API consisting of at least one of "get()", # "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will be # transformed in some way below (by default, to JSON, but may also raise # a cherrypy exception, log itself in a streaming table, etc.). # try: if 'run' in dir(service): # Call the module's run() method, passing it the positional and # keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__[method.lower()] if restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful: result = restfunc(*pargs, **kwargs) else: result = tangelo.HTTPStatusCode(405, "Method not allowed") except Exception as e: bt = traceback.format_exc() tangelo.log("Caught exception while executing service %s" % (tangelo.request_path()), "SERVICE") tangelo.log(bt, "SERVICE") result = tangelo.HTTPStatusCode("501 Error in Python Service", "There was an error executing service %s:<br><pre>%s</pre>" % (tangelo.request_path(), bt)) # Restore the path to what it was originally. sys.path = origpath # Check the type of the result to decide what result to finally return: # # 1. If it is an HTTPStatusCode object, raise a cherrypy HTTPError # exception, which will cause the browser to do the right thing. # # 2. TODO: If it's a Python generator object, log it with the Tangelo # streaming API. # # 3. If it's a Python dictionary, convert it to JSON. # # 4. If it's a string, don't do anything to it. # # This allows the services to return a Python object if they wish, or to # perform custom serialization (such as for MongoDB results, etc.). if isinstance(result, tangelo.HTTPStatusCode): if result.msg: raise cherrypy.HTTPError(result.code, result.msg) else: raise cherrypy.HTTPError(result.code) elif "next" in dir(result): # Generate a key corresponding to this object, using 100 random # bytes from the system - ensure the random key is not already in # the table (even though it would be crazy to wind up with a # collision). # # TODO(choudhury): replace this with a call to generate_key(). # Move the comment above into the generate_key() function. key = md5.md5(os.urandom(100)).hexdigest() while key in self.streams: key = md5.md5(os.urandom(100)).hexdigest() # Log the object in the streaming table. self.streams[key] = result # Create an object describing the logging of the generator object. result = tangelo.empty_response() result["stream_key"] = key # Serialize it to JSON. result = json.dumps(result) elif not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: t = e.message.split("<service.")[1].split()[0] msg = "Service %s returned an object of type %s that could not be serialized to JSON" % (tangelo.request_path(), t) tangelo.log("Error: %s" % (msg), "SERVICE") raise cherrypy.HTTPError("501 Error in Python Service", msg) return result