def delete(key=None): # TODO(choudhury): shut down a vtkweb process by key after a given timeout. processes = tangelo.plugin_store()["processes"] if key is None: tangelo.http_status(400, "Required Argument Missing") return {"error": "'key' argument is required"} # Check for the key in the process table. if key not in processes: tangelo.http_status(400, "Key Not Found") return {"error": "Key %s not in process table" % (key)} # Terminate the process. tangelo.log_info("VTKWEB", "Shutting down process %s" % (key)) proc = processes[key] proc["process"].terminate() proc["process"].wait() tangelo.log_info("VTKWEB", "Process terminated") # Remove the process entry from the table. del processes[key] return {"key": key}
def run(*path, **query): if len(path) == 0: tangelo.http_status(400, "Missing Path") return {"error": "missing path to config file"} required = query.get("required") is not None url = "/" + "/".join(path) content = analyze_url(url).content if content is None or content.type not in [Content.File, Content.NotFound]: tangelo.http_status(400, "Illegal Path") return {"error": "illegal web path (path does not point to a config file)"} elif content.type == Content.NotFound: if required: return {"error": "File not found", "file": url} else: return {"result": {}} try: config = tangelo.util.yaml_safe_load(content.path, dict) except IOError: tangelo.http_status(404) return {"error": "could not open file at %s" % (url)} except TypeError: tangelo.http_status(400, "Not An Associative Array") return {"error": "file at %s did not contain a top-level associative array" % (url)} except ValueError as e: tangelo.http_status(400, "YAML Error") return {"error": "could not parse YAML from file at %s: %s" % (url, e.message)} return {"result": config}
def get(key=None): processes = tangelo.plugin_store()["processes"] # If no key was supplied, return list of running processes. if key is None: return processes.keys() # Error for bad key. if key not in processes: tangelo.http_status(400, "No Such Process Key") return {"error": "Requested key not in process table"} # Retrieve the process entry. rec = processes[key] response = {"status": "complete", "process": "running", "port": rec["port"], "stdout": rec["stdout"].readlines(), "stderr": rec["stderr"].readlines()} # Check the status of the process. returncode = rec["process"].poll() if returncode is not None: # Since the process has ended, delete the process object. del processes[key] # Fill out the report response. response["process"] = "terminated" response["returncode"] = returncode return response
def edges(m, id="", key=None, value=None, **kwargs): def convert(mongoEdge): edge = { k: str(v) if isinstance(v, ObjectId) else v for k, v in mongoEdge.get("data", {}).iteritems() } edge["_id"] = str(mongoEdge["_id"]) edge["_type"] = "edge" edge["_inV"] = str(mongoEdge["source"]) edge["_outV"] = str(mongoEdge["target"]) return edge if id == "" and key is None and value is None: # Return all edges. results = map(convert, m.find({"type": "link"})) return { "version": "*.*", "results": results, "totalSize": len(results), "queryTime": 0.0 } tangelo.http_status(501) return {"error": "unimplemented"}
def get(key=None): # If no key was supplied, return list of running processes. if key is None: return processes.keys() # Error for bad key. if key not in processes: tangelo.http_status(400, "No Such Process Key") return {"error": "Requested key not in process table"} # Retrieve the process entry. rec = processes[key] response = {"status": "complete", "process": "running", "port": rec["port"], "stdout": rec["stdout"].readlines(), "stderr": rec["stderr"].readlines()} # Check the status of the process. returncode = rec["process"].poll() if returncode is not None: # Since the process has ended, delete the process object. del processes[key] # Fill out the report response. response["process"] = "terminated" response["returncode"] = returncode return response
def run(host=None, db=None, coll=None, key=""): # Connect to the Mongo collection client = MongoClient(host) db = client[db] graph = db[coll] # Find the target node; make sure it (1) exists and (2) is a node. oid = ObjectId(key) victim = graph.find_one({"_id": oid}) if victim is None: tangelo.http_status(400, "Bad Argument") return {"error": "No node with key %s" % (key)} if victim.get("type") != "node": tangelo.http_status(400, "Bad Argument") return {"error": "Document with key %s is not a node" % (key), "document": json.loads(bson.json_util.dumps(victim))} # Remove all links associated to the target node. graph.remove({"type": "link", "$or": [{"source": oid}, {"target": oid}]}) # Remove the node itself. graph.remove({"_id": oid}) return "OK"
def run(host=None, db=None, coll=None, key=""): # Connect to the Mongo collection client = MongoClient(host) db = client[db] graph = db[coll] # Find the target node; make sure it (1) exists and (2) is a node. oid = ObjectId(key) victim = graph.find_one({"_id": oid}) if victim is None: tangelo.http_status(400, "Bad Argument") return {"error": "No link with key %s" % (key)} if victim.get("type") != "link": tangelo.http_status(400, "Bad Argument") return { "error": "Document with key %s is not a link" % (key), "document": json.loads(bson.json_util.dumps(victim)) } # Remove the link itself. graph.remove({"_id": oid}) return "OK"
def vertices(m, id="", op=None, key=None, value=None, **kwargs): if id != "": # Return the requested vertex vertex = m.find_one({"_id": ObjectId(id)}) if vertex is None or vertex["type"] != "node": tangelo.http_status(404) return {"error": "No such vertex"} response = {"version": "*.*", "results": {"_type": "vertex", "_id": id}, "queryTime": 0.0} for key, value in vertex.get("data", {}).iteritems(): response["results"][key] = value return response else: def process(rec): result = {"_type": "vertex", "_id": str(rec["_id"])} result.update({k: v for k, v in rec.get("data", {}).iteritems()}) return result vertices = map(process, m.find({"type": "node"})) response = {"version": "*.*", "results": vertices, "totalSize": len(vertices), "queryTime": 0.0} return response
def post(op='', **kwargs): if op == '': return createPipeline(**kwargs) elif op == 'execute': return executePipeline(**kwargs) elif op == 'export': return exportPipeline(**kwargs) else: tangelo.http_status(404)
def get_stub(): server_channel_address = os.environ.get('TA2_SERVER_CONN') # complain in the return if we didn't get an address to connect to if server_channel_address is None: tangelo.http_status(500) return {'error': 'TA2_SERVER_CONN environment variable is not set!'} channel = grpc.insecure_channel(server_channel_address) stub = core_pb2_grpc.CoreStub(channel) return stub
def plugin(self, *path, **args): # Refresh the plugin registry. if self.plugins: error = self.plugins.refresh() if error is not None: tangelo.content_type("text/plain") tangelo.http_status(400, "Bad Plugin Configuration") return error return self.execute_analysis(args)
def delete(key=None): if key is None: tangelo.http_status(400, "Stream Key Required") return {"error": "No stream key was specified"} elif key not in streams: tangelo.http_status(404, "No Such Stream Key") return {"error": "Key '%s' does not correspond to an active stream" % (key)} else: del streams[key] return {"key": key}
def run(op,*args,**kwargs): if op == 'data': return getDataset() elif op == 'listfeatures': return listDatasetFeatures() elif op == 'metadata': return listFeatureMetadata() elif op == 'problems': return getProblems() else: tangelo.http_status(404) return 'illegal operation "%s"' % (op) if op else 'missing operation'
def run(op, *args, **kwargs): if op == 'list': return listDatasets() elif op == 'listtesting': return listTestingDataset() elif op == 'data': return getDataset(*args, **kwargs) elif op == 'datatesting': return getTestingDataset() else: tangelo.http_status(404) return 'illegal operation "%s"' % (op) if op else 'missing operation'
def get(mainOp, graph, op, *args, **qargs): if mainOp != "graphs": tangelo.http_status(400) return {"error": "'%s' is not a legal method" % (mainOp)} graph_path = graph.split(",") if len(graph_path) == 2: host = "localhost" db = graph_path[0] coll = graph_path[1] elif len(graph_path) == 3: host = graph_path[0] db = graph_path[1] coll = graph_path[2] else: tangelo.http_status(400) return {"error": "Invalid graph name %s" % (graph)} try: m = MongoClient(host)[db][coll] except ConnectionFailure: tangelo.http_status(404) return {"error": "Could not connect to graph %s" % (graph)} if op == "vertices": return vertices(m, *args, **qargs) elif op == "edges": return edges(m, *args, **qargs) else: tangelo.http_status(501) return {"error": "Operation %s not implemented"}
def verifyDomainId(**kwargs): if 'team_id' not in kwargs or 'domain_id' not in kwargs: tangelo.http_status(500) tangelo.log("team_id and domain_id required.") return "team id and domain id required for this call." team_id = int(kwargs['team_id']) domain_id = int(kwargs['domain_id']) if not datawake_mysql.hasDomains(team_id,domain_id): tangelo.http_status(401) tangelo.log("401 Unauthorized. Team has no access to requested domain") return "401 Unauthorized" return callback(**kwargs)
def verifyDomainId(**kwargs): if 'team_id' not in kwargs or 'domain_id' not in kwargs: tangelo.http_status(500) tangelo.log("team_id and domain_id required.") return "team id and domain id required for this call." team_id = int(kwargs['team_id']) domain_id = int(kwargs['domain_id']) if not datawake_mysql.hasDomains(team_id, domain_id): tangelo.http_status(401) tangelo.log( "401 Unauthorized. Team has no access to requested domain") return "401 Unauthorized" return callback(**kwargs)
def verifyTeamId(**kwargs): if 'team_id' not in kwargs: tangelo.http_status(500) tangelo.log("team_id required.") return "team id required for this call." user = get_user() # verify the user can access the team if not datawake_mysql.hasTeamAccess(user.get_email(),kwargs['team_id']): tangelo.content_type() tangelo.http_status(401) tangelo.log("401 Unauthorized. User has no access to requested team.") return "401 Unauthorized" return callback(**kwargs)
def copyToWebRoot(returnRec=None): resultURI = returnRec['resultUri'] print 'copying pipelineURI:', resultURI if resultURI is None: tangelo.http_status(500) return {'error': 'no resultURI for executed pipeline'} if resultURI[0:7] == 'file://': resultURI = resultURI[7:] # copy the results file under the webroot so it can be read by # javascript without having cross origin problems shutil.copy(resultURI, 'pipelines') print 'copy completed' return resultURI
def stream_start(url, kwargs): content = tangelo.server.analyze_url(url).content if content is None or content.type != Content.Service: tangelo.http_status(500, "Error Opening Streaming Service") return {"error": "could not open streaming service"} else: # Extract the path to the service and the list of positional # arguments. module_path = content.path pargs = content.pargs # Get the service module. try: service = modules.get(module_path) except: tangelo.http_status(501, "Error Importing Streaming Service") tangelo.content_type("application/json") return tangelo.util.traceback_report( error="Could not import module %s" % (module_path)) else: # Check for a "stream" function inside the module. if "stream" not in dir(service): tangelo.http_status(400, "Non-Streaming Service") return { "error": "The requested streaming service does not implement a 'stream()' function" } else: # Call the stream function and capture its result. try: stream = service.stream(*pargs, **kwargs) except Exception: result = tangelo.util.traceback_report( error= "Caught exception during streaming service execution", module=tangelo.request_path()) tangelo.log_warning( "STREAM", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join( result["traceback"]))) tangelo.http_status(500, "Streaming Service Raised Exception") tangelo.content_type("application/json") return result else: # Generate a key corresponding to this object. key = tangelo.util.generate_key(streams) # Log the object in the streaming table. streams[key] = stream # Create an object describing the logging of the generator object. return {"key": key}
def getDataset(name): global dataRoot dataPath = os.path.abspath(os.path.join(dataRoot, name, 'data')) datafile = os.path.join(dataPath, 'trainData.csv') targetfile = os.path.join(dataPath, 'trainTargets.csv') try: reader = csv.reader(open(datafile)) except IOError: tangelo.http_status(500) return {'error': 'Could not open datafile for dataset %s' % (name)} try: treader = csv.reader(open(targetfile)) except IOError: tangelo.http_status(500) return { 'error': 'Could not open training data for dataset %s' % (name) } rows = list(reader) trows = list(treader) # read in the initial training data dicts = [] for row in rows[1:]: dicts.append({k: promote(v) for k, v in zip(rows[0], row)}) # read the training target variable tdicts = [] for trow in trows[1:]: tdicts.append({k: promote(v) for k, v in zip(trows[0], trow)}) trainName = tdicts[0].keys()[1] # add the training variable to the datatable for row in dicts: indexToFix = row['d3mIndex'] row[trainName] = returnMatchTrain(indexToFix, tdicts, trainName) schemaFile = os.path.join(dataPath, 'dataSchema.json') schema = None try: with open(schemaFile) as f: schema = json.loads(f.read()) except IOError: tangelo.http_status(500) return {'error': 'Could not open schemafile for dataset %s' % (name)} except ValueError as e: tangelo.http_status(500) return { 'error': 'Error while parsing schemafile for dataset %s: %s' % (name, e) } return {'data': dicts, 'meta': schema, 'path': dataPath, 'name': name}
def edges(m, id="", key=None, value=None, **kwargs): def convert(mongoEdge): edge = {k: str(v) if isinstance(v, ObjectId) else v for k, v in mongoEdge.get("data", {}).iteritems()} edge["_id"] = str(mongoEdge["_id"]) edge["_type"] = "edge" edge["_inV"] = str(mongoEdge["source"]) edge["_outV"] = str(mongoEdge["target"]) return edge if id == "" and key is None and value is None: # Return all edges. results = map(convert, m.find({"type": "link"})) return {"version": "*.*", "results": results, "totalSize": len(results), "queryTime": 0.0} tangelo.http_status(501) return {"error": "unimplemented"}
def verifyTeamId(**kwargs): if 'team_id' not in kwargs: tangelo.http_status(500) tangelo.log("team_id required.") return "team id required for this call." user = get_user() # verify the user can access the team if not datawake_mysql.hasTeamAccess(user.get_email(), kwargs['team_id']): tangelo.content_type() tangelo.http_status(401) tangelo.log( "401 Unauthorized. User has no access to requested team.") return "401 Unauthorized" return callback(**kwargs)
def forward(project, obj): """Forward an event object to the configured buildbot instance.""" auth = None if projects.get('user') and projects.get('password'): auth = (projects['user'], projects['password']) resp = requests.post( project['buildbot'].rstrip('/') + '/change_hook/github', data={"payload": obj}, auth=auth ) # headers={'CONTENT-TYPE': 'application/x-www-form-urlencoded'} if resp.ok: tangelo.http_status(200, 'OK') return 'OK' else: tangelo.http_status(400, "Bad project configuration") return 'Bad project configuration'
def get(team_id): """ Verify the logged in user has access to the requested team and return that teams list of domains. :param team_id: :return: List of domain objects for the team [{'id':domain_id,'name','name':domainname,'description':domaindescription},..] """ user = helper.get_user() if not db.hasTeamAccess(user.get_email(),team_id): tangelo.content_type() tangelo.http_status(401) tangelo.log("401 Unauthorized domains/?team_id="+str(team_id)+" user: "******"401 Unauthorized" else: results = db.get_domains(team_id) return json.dumps(results)
def get(team_id): """ Verify the logged in user has access to the requested team and return that teams list of domains. :param team_id: :return: List of domain objects for the team [{'id':domain_id,'name','name':domainname,'description':domaindescription},..] """ user = helper.get_user() if not db.hasTeamAccess(user.get_email(), team_id): tangelo.content_type() tangelo.http_status(401) tangelo.log("401 Unauthorized domains/?team_id=" + str(team_id) + " user: "******"401 Unauthorized" else: results = db.get_domains(team_id) return json.dumps(results)
def add_trail(team_id,domain_id,name,description=''): tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id)) #check the trail name if name is None or len(name) < 1: raise ValueError("Trail names must have at least one character") user = helper.get_user() # create and then return the new trail try: newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email()) except Exception as e: tangelo.log(e) tangelo.http_status(501) return "Failed to create trail. This trail name may already be used." newTrail = dict(id=newTrailId,name=name,description=description) tangelo.log(newTrail) return json.dumps(newTrail)
def add_trail(team_id,domain_id,name,description=''): tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id)) #check the trail name if name is None or len(name) < 1: raise ValueError("Trail names must have at least one character") user = helper.get_user() # create and then return the new trail try: newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email()) newTrail = dict(id=newTrailId,name=name,description=description) tangelo.log(newTrail) return json.dumps(newTrail) except Exception as e: tangelo.log(e) tangelo.http_status(501) return "Failed to create trail. This trail name may already be used."
def stream_next(key): if key not in streams: tangelo.http_status(404, "No Such Key") return {"error": "Stream key does not correspond to an active stream", "stream": key} else: # Grab the stream in preparation for running it. stream = streams[key] # Attempt to run the stream via its next() method - if this # yields a result, then continue; if the next() method raises # StopIteration, then there are no more results to retrieve; if # any other exception is raised, this is treated as an error. try: return stream.next() except StopIteration: del streams[key] tangelo.http_status(204, "Stream Finished") return "OK" except: del streams[key] tangelo.http_status(500, "Streaming Service Exception") tangelo.content_type("application/json") error_code = tangelo.util.generate_error_code() tangelo.util.log_traceback("STREAM", error_code, "Offending stream key: %s" % (key), "Uncaught executing executing service %s" % (tangelo.request_path)) return tangelo.util.error_report(error_code)
def stream_next(key): if key not in streams: tangelo.http_status(404, "No Such Key") return { "error": "Stream key does not correspond to an active stream", "stream": key } else: # Grab the stream in preparation for running it. stream = streams[key] # Attempt to run the stream via its next() method - if this # yields a result, then continue; if the next() method raises # StopIteration, then there are no more results to retrieve; if # any other exception is raised, this is treated as an error. try: return stream.next() except StopIteration: del streams[key] tangelo.http_status(204, "Stream Finished") return "OK" except: del streams[key] tangelo.http_status(500, "Streaming Service Exception") tangelo.content_type("application/json") return tangelo.util.traceback_report( error="Caught exception while executing stream service", stream=key)
def stream_start(url, kwargs): content = tangelo.server.analyze_url(url).content if content is None or content.type != Content.Service: tangelo.http_status(500, "Error Opening Streaming Service") return {"error": "could not open streaming service"} else: # Extract the path to the service and the list of positional # arguments. module_path = content.path pargs = content.pargs # Get the service module. try: service = modules.get(module_path) except: tangelo.http_status(500, "Error Importing Streaming Service") tangelo.content_type("application/json") error_code = tangelo.util.generate_error_code() tangelo.util.log_traceback("STREAM", error_code, "Could not import module %s" % (tangelo.request_path())) return tangelo.util.error_report(error_code) else: # Check for a "stream" function inside the module. if "stream" not in dir(service): tangelo.http_status(400, "Non-Streaming Service") return {"error": "The requested streaming service does not implement a 'stream()' function"} else: # Call the stream function and capture its result. try: stream = service.stream(*pargs, **kwargs) except Exception: tangelo.http_status(500, "Streaming Service Raised Exception") tangelo.content_type("application/json") error_code = tangelo.util.generate_error_code() tangelo.util.log_traceback("STREAM", error_code, "Could not execute service %s" % (tangelo.request_path())) return tangelo.util.error_report(error_code) else: # Generate a key corresponding to this object. key = tangelo.util.generate_key(streams) # Log the object in the streaming table. streams[key] = stream # Create an object describing the logging of the generator object. return {"key": key}
def post(port=None): # get the protocol version version = core_pb2.DESCRIPTOR.GetOptions().Extensions[ core_pb2.protocol_version] # get the address from an environment variable. This must be set in the executing shell. # During automated evaluation runs, the environment variable will be set by Kubernetes server_channel_address = os.environ.get('TA2_SERVER_CONN') # complain in the return if we didn't get an address to connect to if server_channel_address is None: tangelo.http_status(500) return {'error': 'TA2_SERVER_CONN environment variable is not set!'} #channel = grpc.insecure_channel('localhost:%d' % (int(port))) channel = grpc.insecure_channel(server_channel_address) stub = core_pb2_grpc.CoreStub(channel) resp = stub.StartSession( core_pb2.SessionRequest(user_agent='modsquad', version=version)) return MessageToJson(resp)
def vertices(m, id="", op=None, key=None, value=None, **kwargs): if id != "": # Return the requested vertex vertex = m.find_one({"_id": ObjectId(id)}) if vertex is None or vertex["type"] != "node": tangelo.http_status(404) return {"error": "No such vertex"} response = { "version": "*.*", "results": { "_type": "vertex", "_id": id }, "queryTime": 0.0 } for key, value in vertex.get("data", {}).iteritems(): response["results"][key] = value return response else: def process(rec): result = {"_type": "vertex", "_id": str(rec["_id"])} result.update({k: v for k, v in rec.get("data", {}).iteritems()}) return result vertices = map(process, m.find({"type": "node"})) response = { "version": "*.*", "results": vertices, "totalSize": len(vertices), "queryTime": 0.0 } return response
def run(method=None, data=None, **kwargs): if data is None: tangelo.http_status(400, 'Required argument missing') return None fn = select_method(method) if fn is None: tangelo.http_status(400, 'Required argument missing') return None data = json.loads(data) dataframe = make_frame(data) if fn == run_quadratic: predictor_variables = kwargs.get('predictor_variables') response = kwargs.get('response') quadratic_variables = kwargs.get('quadratic_variables') if predictor_variables is None or response is None or quadratic_variables is None: tangelo.http_status(400, 'Required argument missing') return None args = map(json.loads, [response, predictor_variables, quadratic_variables]) elif fn in [run_lm, run_loess]: predictor_variables = kwargs.get('predictor_variables') response = kwargs.get('response') if predictor_variables is None or response is None: tangelo.http_status(400, 'Required argument missing') return None args = map(json.loads, [response, predictor_variables]) else: tangelo.http_status(500, 'Impossible condition') return str(fn(dataframe, *args))
def post(*arg, **kwarg): """Listen for github webhooks, authenticate, and forward to buildbot.""" # retrieve the headers from the request try: received = tangelo.request_header('X-Hub-Signature')[5:] except Exception: received = '' # get the request body as a dict # for json body = tangelo.request_body().read() try: obj = json.loads(body) except: tangelo.http_status(400, "Could not load json object") return "Could not load json object" # obj = json.loads(kwarg['payload']) open('last.json', 'w').write(json.dumps(obj, indent=2)) project = get_project(obj.get('repository', {}).get('full_name')) if project is None: tangelo.http_status(400, "Unknown project") return 'Unknown project' # make sure this is a valid request coming from github if not authenticate(project.get('api-key', ''), body, received): tangelo.http_status(403, "Invalid signature") return 'Invalid signature' event = tangelo.request_header('X-Github-Event') if project['events'] == '*' or event in project['events']: obj['event'] = event # add a new item to the test queue return forward(project, body) else: tangelo.http_status(200, "Unhandled event") return 'Unhandled event'
def run(): config_file = os.environ.get('JSON_CONFIG_PATH') print 'environment variable said:', config_file #config_file = "/Users/clisle/proj/D3M/code/eval/config.json" print 'config service: looking for config file..', config_file if config_file is None: tangelo.http_status(500) return {'error': 'JSON_CONFIG_PATH is not set!'} try: with open(config_file) as f: text = f.read() except IOError as e: tangelo.http_status(500) return {'error': str(e)} try: config = json.loads(text) except ValueError as e: tangelo.http_status(500) return {'error': 'Could not parse JSON - %s' % (str(e))} # make place for executed pipelines to be copied (see pipeline.py copyToWebRoot). # ignore exceptions because directory might exist if rerun on new problems without # resetting (should only occur on native execution instead of single problem container execution) try: os.mkdir('pipelines') except: pass print 'received json configuration:', config os.environ['PROBLEM_SCHEMA_PATH'] = config['problem_schema'] os.environ['DATASET_SCHEMA_PATH'] = config['dataset_schema'] os.environ['TRAINING_DATA_ROOT'] = config['training_data_root'] os.environ['PROBLEM_ROOT'] = config['problem_root'] os.environ['EXECUTABLES_ROOT'] = config['executables_root'] # used by the ta2read service os.environ['TEMP_STORAGE_ROOT'] = config['temp_storage_root'] return config
def post(*pathcomp, **kwargs): if len(pathcomp) == 0: # TODO: raise error condition pass action = pathcomp[0] args = pathcomp[1:] if action == "start": if len(args) == 0: tangelo.http_status(400, "Path To Service Required") return {"error": "No service path was specified"} return stream_start("/" + "/".join(args), kwargs) elif action == "next": if len(args) != 1: tangelo.http_status(400, "Stream Key Required") return {"error": "No stream key was specified"} return stream_next(args[0]) else: tangelo.http_status(400, "Illegal POST action") return {"error": "Illegal POST action '%s'" % (action)}
def run(): config_file = os.environ.get('JSON_CONFIG_PATH') if config_file is None: tangelo.http_status(500) return {'error': 'JSON_CONFIG_PATH is not set!'} try: with open(config_file) as f: text = f.read() except IOError as e: tangelo.http_status(500) return {'error': str(e)} try: config = json.loads(text) except ValueError as e: tangelo.http_status(500) return {'error': 'Could not parse JSON - %s' % (str(e))} os.environ['PROBLEM_SCHEMA'] = config['problem_schema'] os.environ['DATASET_SCHEMA'] = config['dataset_schema'] os.environ['TRAINING_DATA_ROOT'] = config['training_data_root'] return config
def run(plot_type, sort=False): if plot_type == "count": return count_barchart(sort=sort) else: tangelo.http_status(400, "Illegal plot type") return {"error": "Illegal plot type: %s" % (plot_type)}
def run(): tangelo.http_status(400, "Bad Configuration") return {"error": msg}
def post(*pargs, **query): args = query.get("args", "") timeout = float(query.get("timeout", 0)) if len(pargs) == 0: tangelo.http_status(400, "Required Argument Missing") return {"error": "No program path was specified"} program_url = "/" + "/".join(pargs) content = analyze_url(program_url).content if content is None or content.type != Content.File: tangelo.http_status(404, "Not Found") return {"error": "Could not find a script at %s" % (program_url)} program = content.path # Check the user arguments. userargs = args.split() if "--port" in userargs: tangelo.http_status(400, "Illegal Argument") return {"error": "You may not specify '--port' among the arguments passed in 'args'"} # Obtain an available port. port = tangelo.util.get_free_port() # Generate a unique key. key = tangelo.util.generate_key(processes.keys()) # Detect http vs. https scheme = "ws" ssl_key = cherrypy.config.get("server.ssl_private_key") ssl_cert = cherrypy.config.get("server.ssl_certificate") # Generate command line. cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs if ssl_key and ssl_cert: scheme = "wss" cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert]) # Launch the requested process. tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline))) try: process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except (OSError, IOError) as e: tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process") return {"error": e.strerror} # Capture the new process's stdout and stderr streams in # non-blocking readers. stdout = tangelo.util.NonBlockingReader(process.stdout) stderr = tangelo.util.NonBlockingReader(process.stderr) # Read from stdout to look for the signal that the process has # started properly. class FactoryStarted: pass class Failed: pass class Timeout: pass signal = "Starting factory" if timeout <= 0: timeout = 10 sleeptime = 0.5 wait = 0 saved_lines = [] try: while True: lines = stdout.readlines() saved_lines += lines for line in lines: if line == "": # This indicates that stdout has closed without # starting the process. raise Failed() elif signal in line: # This means that the server has started. raise FactoryStarted() # If neither failure nor success occurred in the last block # of lines from stdout, either time out, or try again after # a short delay. if wait >= timeout: raise Timeout() wait += sleeptime time.sleep(sleeptime) except Timeout: tangelo.http_status(524, "Timeout") return {"error": "Process startup timed out"} except Failed: tangelo.http_status(500) return {"error": "Process did not start up properly", "stdout": saved_lines, "stderr": stderr.readlines()} except FactoryStarted: stdout.pushlines(saved_lines) # Create a websocket handler path dedicated to this process. host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp") # Log the new process in the process table, including non-blocking # stdout and stderr readers. processes[key] = {"port": port, "process": process, "stdout": stdout, "stderr": stderr} # Form the websocket URL from the hostname/port used in the # request, and the newly generated key. url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key) return {"key": key, "url": url}
def has_session(**kwargs): if 'user' in cherrypy.session: return callback(**kwargs) tangelo.http_status(401) tangelo.log("401 Unauthorized No user in session") return "No User in the current session"
def get_stream_info(key): tangelo.http_status(501) return {"error": "stream info method currently unimplemented"}
def post(*arg, **kwarg): """Listen for github webhooks, authenticate, and forward to buildbot.""" # retrieve the headers from the request print "MASTER RECEIVED A POST EVENT" # print "TGELO CONFI",tangelo.cherrypy.request.header_list try: received = tangelo.request_header('X-Hub-Signature')[5:] except Exception: try: received = tangelo.request_header('BOT-Signature')[5:] except Exception: received = '' # get the request body as a dict # for json body = tangelo.request_body().read() try: obj = json.loads(body) except: tangelo.http_status(400, "Could not load json object") return "Could not load json object" # obj = json.loads(kwarg['payload']) #open('last.json', 'w').write(json.dumps(obj, indent=2)) project_name = obj.get('repository', {}).get('full_name') project = get_project(project_name) if project is None: tangelo.http_status(400, "Unknown project") return 'Unknown project' # make sure this is a valid request coming from github if not authenticate(project.get('api-key', ''), body, received) \ and \ not authenticate(project.get('bot-key', ''), body, received): tangelo.http_status(403, "Invalid signature") return 'Invalid signature' event = tangelo.request_header('X-Github-Event') print "EVENT:",event if project['github-events'] == '*' or event in project['github-events']: obj['event'] = event try: commit = obj["head_commit"] is_commit = True except: commit = obj["pull_request"] is_commit = False try: act = obj["action"] if act == "closed": # closed PR no need to run anything return "Closed PR, skipping" except: pass if commit is None: ## no head_Commit trying to see if it's a pull request return "Null Head Commit Found, not a PR either skipping" if is_commit: commit_id = commit["id"] commit_msg = commit["message"] else: ## It 's a PR faking the head_commit/id bits for slaves commits_url = commit["commits_url"] commit_id = commit["head"]["sha"] commit_statuses_url=commit["statuses_url"] commit_ref = commit["head"]["ref"] resp = requests.get(commits_url,verify=False) commit = resp.json()[-1]["commit"] commit_msg=commit["message"] commit["id"]=commit_id obj["ref"]=commit_ref commit["statuses_url"]=commit_statuses_url obj["head_commit"]=commit signature = hmac.new(str(project["bot-key"]), json.dumps(obj), hashlib.sha1).hexdigest() if commit_msg.find("##bot##skip-commit")>-1: # User requested to not send this commit to bots return "Skipped testing commit '%s' at committer request (found string '##bot##skip-commit')" nok = 0 for islave, slave in enumerate(project["slaves"]): islaves = commit_msg.find("##bot##skip-slaves") if islaves>-1: # ok some slaves need skipping msg = commit_msg[islaves+18:] iend = msg.find("\n") msg = msg[:iend].strip().split() iskip = False for m in msg: if slave.find(m)>-1: iskip = True break if iskip: print "\033[%im" % (91+islave),"Commit asked to skip:",slave,"\033[0m" nok+=1 continue print "\033[%im" % (91+islave),"SENDING TO:",slave,"\033[0m" try: resp = forward(slave,obj,signature) if resp.ok: nok+=1 except: print "\033[%im" % (91+islave),"could not connect","\033[0m" nok+=1 if nok>0: return "Ok sent this to %i slaves out of %i" % (nok,len(project["slaves"])) else: msg = "All slaves failed to respond, last error was: %s" % resp.text print msg tangelo.http_status(resp.status_code, msg) return msg elif tangelo.request_header('BOT-Event') == "status": ## put here code to update status of commit on github headers = { "Authorization":"token "+project["token"], } commit_id = obj["commit"]["id"] if obj["code"] == 0: state = "success" elif obj["code"] is None: state = "pending" else: state = "failure" slave = obj["slave_host"] try: islave = project["slaves"].find("http://"+slave) except: islave = -91 # Turn off styling pth = os.path.join(project["logs_dir"],slave,project_name,commit_id) print "\033[%im" % (91+islave),"DUMPING INFO IN:",pth,"\033[0m" print "\033[%im" % (91+islave),"could not connect","\033[0m" if not os.path.exists(str(pth)): print "Creating:",pth os.makedirs(pth) f=open(os.path.join(pth,cmd2str(obj["command"])),"w") print >>f,"<html><body>" print >>f,"<h1>%s (%s)</h1><br><h2>commit: %s<h2>" % (project_name,obj["slave_name"],commit_id) host = tangelo.cherrypy.url() host=host[host.find("//")+2:] if obj["previous"] is not None: ptarget = "http://%s/%s/%s/%s/%s" % (host,slave,project_name,commit_id,cmd2str(obj["previous"])) print >>f, "<h2>PREVIOUS COMMAND</h2>" print >>f,"<a href='%s'>" % ptarget,obj["previous"],"</a>" print >>f, "<h2>COMMAND</h2>" print >>f,"<pre>",obj["command"],"</pre>" if obj["command"].find("ctest")>-1: print >>f, "<h3>CTEST PAGE</h3>" build_name = "%s-%s" % (slave.replace(":",""),commit_id) ptarget = "https://open.cdash.org/index.php?compare1=65&filtercount=2&field1=buildname%%2Fstring&project=UV-CDAT&field2=buildstarttime%%2Fdate&value1=%s" % build_name print >>f,"<A HREF='%s'>Click here</A>" % ptarget print >>f, "<h3>OUTPUT</h3>" print >>f,"<pre>",obj["output"],"</pre>" print >>f, "<h3>ERROR</h3>" print >>f,"<pre>",obj["error"],"</pre>" print >>f,"</body></html>" f.close() target = "http://%s/%s/%s/%s/%s" % (host,slave,project_name,commit_id,cmd2str(obj["command"])) context = "cont-int/LLNL/%s-%s" % (obj["os"],obj["slave_name"]) data = { "state":state, "target_url": target, "description": "'%s' (%s)" % (obj["command"][:20],time.asctime()), "context": context, } resp = requests.post( obj["commit"]["statuses_url"].replace("{sha}",obj["commit"]["id"]), data = json.dumps(data), verify = False, headers = headers) return "Received and treated a BOT STATUS update event" else: tangelo.http_status(200, "Unhandled event") return 'Unhandled event'
def invoke_service(self, module, *pargs, **kwargs): # TODO(choudhury): This method should attempt to load the named module, # then invoke it with the given arguments. However, if the named # module is "config" or something similar, the method should instead # launch a special "config" app, which lists the available app modules, # along with docstrings or similar. It should also allow the user to # add/delete search paths for other modules. tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function) - it will be modified before invoking the service, and must # be restored afterwards. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Extend the system path with the module's home path. sys.path.insert(0, modpath) try: service = self.modules.get(module) except: tangelo.http_status(501, "Error Importing Service") tangelo.content_type("application/json") result = {"error": "Could not import module %s" % (tangelo.request_path()), "traceback": traceback.format_exc().split("\n")} else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if 'run' in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__.get(method.lower()) if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: tangelo.http_status(405, "Method Not Allowed") tangelo.content_type("application/json") result = {"error": "Method '%s' is not allowed in this service" % (method)} except: stacktrace = traceback.format_exc() tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), stacktrace)) tangelo.http_status(501, "Web Service Error") tangelo.content_type("application/json") result = {"error": "Error executing service", "module": tangelo.request_path(), "traceback": stacktrace.split("\n")} # Restore the path to what it was originally. sys.path = origpath # If the result is not a string, attempt to convert it to one via JSON # serialization. This allows services to return a Python object if they # wish, or to perform custom serialization (such as for MongoDB results, # etc.). if not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: tangelo.http_status(400, "JSON Error") tangelo.content_type("application/json") result = {"error": "JSON type error executing service", "message": e.message} else: tangelo.content_type("application/json") return result
def invoke_service(self, module, *pargs, **kwargs): tangelo.content_type("text/plain") # Save the system path (be sure to *make a copy* using the list() # function). This will be restored to undo any modification of the path # done by the service. origpath = list(sys.path) # By default, the result should be an object with error message in if # something goes wrong; if nothing goes wrong this will be replaced # with some other object. result = {} # Store the modpath in the thread-local storage (tangelo.paths() makes # use of this per-thread data, so this is the way to get the data # across the "module boundary" properly). modpath = os.path.dirname(module) cherrypy.thread_data.modulepath = modpath cherrypy.thread_data.modulename = module # Change the current working directory to that of the service module, # saving the old one. This is so that the service function executes as # though it were a Python program invoked normally, and Tangelo can # continue running later in whatever its original CWD was. save_cwd = os.getcwd() os.chdir(modpath) try: service = self.modules.get(module) except: tangelo.http_status(501, "Error Importing Service") tangelo.content_type("application/json") result = tangelo.util.traceback_report(error="Could not import module %s" % (tangelo.request_path())) else: # Try to run the service - either it's in a function called # "run()", or else it's in a REST API consisting of at least one of # "get()", "put()", "post()", or "delete()". # # Collect the result in a variable - depending on its type, it will # be transformed in some way below (by default, to JSON, but may # also raise a cherrypy exception, log itself in a streaming table, # etc.). try: if "run" in dir(service): # Call the module's run() method, passing it the positional # and keyword args that came into this method. result = service.run(*pargs, **kwargs) else: # Reaching here means it's a REST API. Check for the # requested method, ensure that it was marked as being part # of the API, and call it; or give a 405 error. method = cherrypy.request.method restfunc = service.__dict__.get(method.lower()) if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful): result = restfunc(*pargs, **kwargs) else: tangelo.http_status(405, "Method Not Allowed") tangelo.content_type("application/json") result = {"error": "Method '%s' is not allowed in this service" % (method)} except: tangelo.http_status(501, "Web Service Error") tangelo.content_type("application/json") result = tangelo.util.traceback_report(error="Error executing service", module=tangelo.request_path()) tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join(result["traceback"]))) # Restore the path to what it was originally. sys.path = origpath # Restore the CWD to what it was before the service invocation. os.chdir(save_cwd) # If the result is not a string, attempt to convert it to one via JSON # serialization. This allows services to return a Python object if they # wish, or to perform custom serialization (such as for MongoDB results, # etc.). if not isinstance(result, types.StringTypes): try: result = json.dumps(result) except TypeError as e: tangelo.http_status(400, "JSON Error") tangelo.content_type("application/json") result = {"error": "JSON type error executing service", "message": e.message} else: tangelo.content_type("application/json") return result
def post(*pargs, **query): args = query.get("args", "") timeout = float(query.get("timeout", 0)) processes = tangelo.plugin_store()["processes"] if len(pargs) == 0: tangelo.http_status(400, "Required Argument Missing") return {"error": "No program path was specified"} program_url = "/" + "/".join(pargs) content = analyze_url(program_url).content if content is None or content.type != Content.File: tangelo.http_status(404, "Not Found") return {"error": "Could not find a script at %s" % (program_url)} elif content.path is None: tangelo.http_status(403, "Restricted") return {"error": "The script at %s is access-restricted"} program = content.path # Check the user arguments. userargs = args.split() if "--port" in userargs: tangelo.http_status(400, "Illegal Argument") return {"error": "You may not specify '--port' among the arguments passed in 'args'"} # Obtain an available port. port = tangelo.util.get_free_port() # Generate a unique key. key = tangelo.util.generate_key(processes.keys()) # Detect http vs. https scheme = "ws" ssl_key = cherrypy.config.get("server.ssl_private_key") ssl_cert = cherrypy.config.get("server.ssl_certificate") # Generate command line. cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs if ssl_key and ssl_cert: scheme = "wss" cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert]) # Launch the requested process. tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline))) try: process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except (OSError, IOError) as e: tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process") return {"error": e.strerror} # Capture the new process's stdout and stderr streams in # non-blocking readers. stdout = tangelo.util.NonBlockingReader(process.stdout) stderr = tangelo.util.NonBlockingReader(process.stderr) # Read from stdout to look for the signal that the process has # started properly. class FactoryStarted: pass class Failed: pass class Timeout: pass signal = "Starting factory" if timeout <= 0: timeout = 10 sleeptime = 0.5 wait = 0 saved_lines = [] try: while True: lines = stdout.readlines() saved_lines += lines for line in lines: if line == "": # This indicates that stdout has closed without # starting the process. raise Failed() elif signal in line: # This means that the server has started. raise FactoryStarted() # If neither failure nor success occurred in the last block # of lines from stdout, either time out, or try again after # a short delay. if wait >= timeout: raise Timeout() wait += sleeptime time.sleep(sleeptime) except Timeout: tangelo.http_status(524, "Timeout") return {"error": "Process startup timed out"} except Failed: tangelo.http_status(500) return {"error": "Process did not start up properly", "stdout": saved_lines, "stderr": stderr.readlines()} except FactoryStarted: stdout.pushlines(saved_lines) # Create a websocket handler path dedicated to this process. host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp") # Log the new process in the process table, including non-blocking # stdout and stderr readers. processes[key] = {"port": port, "process": process, "stdout": stdout, "stderr": stderr} # Form the websocket URL from the hostname/port used in the # request, and the newly generated key. url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key) return {"key": key, "url": url}