def delete(key=None):
# TODO(choudhury): shut down a vtkweb process by key after a given timeout.
processes = tangelo.plugin_store()["processes"]
if key is None:
tangelo.http_status(400, "Required Argument Missing")
return {"error": "'key' argument is required"}
# Check for the key in the process table.
if key not in processes:
tangelo.http_status(400, "Key Not Found")
return {"error": "Key %s not in process table" % (key)}
# Terminate the process.
tangelo.log_info("VTKWEB", "Shutting down process %s" % (key))
proc = processes[key]
proc["process"].terminate()
proc["process"].wait()
tangelo.log_info("VTKWEB", "Process terminated")
# Remove the process entry from the table.
del processes[key]
return {"key": key}
def run(*path, **query):
if len(path) == 0:
tangelo.http_status(400, "Missing Path")
return {"error": "missing path to config file"}
required = query.get("required") is not None
url = "/" + "/".join(path)
content = analyze_url(url).content
if content is None or content.type not in [Content.File, Content.NotFound]:
tangelo.http_status(400, "Illegal Path")
return {"error": "illegal web path (path does not point to a config file)"}
elif content.type == Content.NotFound:
if required:
return {"error": "File not found",
"file": url}
else:
return {"result": {}}
try:
config = tangelo.util.yaml_safe_load(content.path, dict)
except IOError:
tangelo.http_status(404)
return {"error": "could not open file at %s" % (url)}
except TypeError:
tangelo.http_status(400, "Not An Associative Array")
return {"error": "file at %s did not contain a top-level associative array" % (url)}
except ValueError as e:
tangelo.http_status(400, "YAML Error")
return {"error": "could not parse YAML from file at %s: %s" % (url, e.message)}
return {"result": config}
def get(key=None):
processes = tangelo.plugin_store()["processes"]
# If no key was supplied, return list of running processes.
if key is None:
return processes.keys()
# Error for bad key.
if key not in processes:
tangelo.http_status(400, "No Such Process Key")
return {"error": "Requested key not in process table"}
# Retrieve the process entry.
rec = processes[key]
response = {"status": "complete",
"process": "running",
"port": rec["port"],
"stdout": rec["stdout"].readlines(),
"stderr": rec["stderr"].readlines()}
# Check the status of the process.
returncode = rec["process"].poll()
if returncode is not None:
# Since the process has ended, delete the process object.
del processes[key]
# Fill out the report response.
response["process"] = "terminated"
response["returncode"] = returncode
return response
def edges(m, id="", key=None, value=None, **kwargs):
def convert(mongoEdge):
edge = {
k: str(v) if isinstance(v, ObjectId) else v
for k, v in mongoEdge.get("data", {}).iteritems()
}
edge["_id"] = str(mongoEdge["_id"])
edge["_type"] = "edge"
edge["_inV"] = str(mongoEdge["source"])
edge["_outV"] = str(mongoEdge["target"])
return edge
if id == "" and key is None and value is None:
# Return all edges.
results = map(convert, m.find({"type": "link"}))
return {
"version": "*.*",
"results": results,
"totalSize": len(results),
"queryTime": 0.0
}
tangelo.http_status(501)
return {"error": "unimplemented"}
def get(key=None):
# If no key was supplied, return list of running processes.
if key is None:
return processes.keys()
# Error for bad key.
if key not in processes:
tangelo.http_status(400, "No Such Process Key")
return {"error": "Requested key not in process table"}
# Retrieve the process entry.
rec = processes[key]
response = {"status": "complete",
"process": "running",
"port": rec["port"],
"stdout": rec["stdout"].readlines(),
"stderr": rec["stderr"].readlines()}
# Check the status of the process.
returncode = rec["process"].poll()
if returncode is not None:
# Since the process has ended, delete the process object.
del processes[key]
# Fill out the report response.
response["process"] = "terminated"
response["returncode"] = returncode
return response
def run(host=None, db=None, coll=None, key=""):
# Connect to the Mongo collection
client = MongoClient(host)
db = client[db]
graph = db[coll]
# Find the target node; make sure it (1) exists and (2) is a node.
oid = ObjectId(key)
victim = graph.find_one({"_id": oid})
if victim is None:
tangelo.http_status(400, "Bad Argument")
return {"error": "No node with key %s" % (key)}
if victim.get("type") != "node":
tangelo.http_status(400, "Bad Argument")
return {"error": "Document with key %s is not a node" % (key),
"document": json.loads(bson.json_util.dumps(victim))}
# Remove all links associated to the target node.
graph.remove({"type": "link",
"$or": [{"source": oid},
{"target": oid}]})
# Remove the node itself.
graph.remove({"_id": oid})
return "OK"
def run(host=None, db=None, coll=None, key=""):
# Connect to the Mongo collection
client = MongoClient(host)
db = client[db]
graph = db[coll]
# Find the target node; make sure it (1) exists and (2) is a node.
oid = ObjectId(key)
victim = graph.find_one({"_id": oid})
if victim is None:
tangelo.http_status(400, "Bad Argument")
return {"error": "No link with key %s" % (key)}
if victim.get("type") != "link":
tangelo.http_status(400, "Bad Argument")
return {
"error": "Document with key %s is not a link" % (key),
"document": json.loads(bson.json_util.dumps(victim))
}
# Remove the link itself.
graph.remove({"_id": oid})
return "OK"
def vertices(m, id="", op=None, key=None, value=None, **kwargs):
if id != "":
# Return the requested vertex
vertex = m.find_one({"_id": ObjectId(id)})
if vertex is None or vertex["type"] != "node":
tangelo.http_status(404)
return {"error": "No such vertex"}
response = {"version": "*.*", "results": {"_type": "vertex", "_id": id}, "queryTime": 0.0}
for key, value in vertex.get("data", {}).iteritems():
response["results"][key] = value
return response
else:
def process(rec):
result = {"_type": "vertex", "_id": str(rec["_id"])}
result.update({k: v for k, v in rec.get("data", {}).iteritems()})
return result
vertices = map(process, m.find({"type": "node"}))
response = {"version": "*.*", "results": vertices, "totalSize": len(vertices), "queryTime": 0.0}
return response
def post(op='', **kwargs):
if op == '':
return createPipeline(**kwargs)
elif op == 'execute':
return executePipeline(**kwargs)
elif op == 'export':
return exportPipeline(**kwargs)
else:
tangelo.http_status(404)
def get_stub():
server_channel_address = os.environ.get('TA2_SERVER_CONN')
# complain in the return if we didn't get an address to connect to
if server_channel_address is None:
tangelo.http_status(500)
return {'error': 'TA2_SERVER_CONN environment variable is not set!'}
channel = grpc.insecure_channel(server_channel_address)
stub = core_pb2_grpc.CoreStub(channel)
return stub
def plugin(self, *path, **args):
# Refresh the plugin registry.
if self.plugins:
error = self.plugins.refresh()
if error is not None:
tangelo.content_type("text/plain")
tangelo.http_status(400, "Bad Plugin Configuration")
return error
return self.execute_analysis(args)
def delete(key=None):
if key is None:
tangelo.http_status(400, "Stream Key Required")
return {"error": "No stream key was specified"}
elif key not in streams:
tangelo.http_status(404, "No Such Stream Key")
return {"error": "Key '%s' does not correspond to an active stream" % (key)}
else:
del streams[key]
return {"key": key}
def run(op,*args,**kwargs):
if op == 'data':
return getDataset()
elif op == 'listfeatures':
return listDatasetFeatures()
elif op == 'metadata':
return listFeatureMetadata()
elif op == 'problems':
return getProblems()
else:
tangelo.http_status(404)
return 'illegal operation "%s"' % (op) if op else 'missing operation'
def run(op, *args, **kwargs):
if op == 'list':
return listDatasets()
elif op == 'listtesting':
return listTestingDataset()
elif op == 'data':
return getDataset(*args, **kwargs)
elif op == 'datatesting':
return getTestingDataset()
else:
tangelo.http_status(404)
return 'illegal operation "%s"' % (op) if op else 'missing operation'
def get(mainOp, graph, op, *args, **qargs):
if mainOp != "graphs":
tangelo.http_status(400)
return {"error": "'%s' is not a legal method" % (mainOp)}
graph_path = graph.split(",")
if len(graph_path) == 2:
host = "localhost"
db = graph_path[0]
coll = graph_path[1]
elif len(graph_path) == 3:
host = graph_path[0]
db = graph_path[1]
coll = graph_path[2]
else:
tangelo.http_status(400)
return {"error": "Invalid graph name %s" % (graph)}
try:
m = MongoClient(host)[db][coll]
except ConnectionFailure:
tangelo.http_status(404)
return {"error": "Could not connect to graph %s" % (graph)}
if op == "vertices":
return vertices(m, *args, **qargs)
elif op == "edges":
return edges(m, *args, **qargs)
else:
tangelo.http_status(501)
return {"error": "Operation %s not implemented"}
def verifyDomainId(**kwargs):
if 'team_id' not in kwargs or 'domain_id' not in kwargs:
tangelo.http_status(500)
tangelo.log("team_id and domain_id required.")
return "team id and domain id required for this call."
team_id = int(kwargs['team_id'])
domain_id = int(kwargs['domain_id'])
if not datawake_mysql.hasDomains(team_id,domain_id):
tangelo.http_status(401)
tangelo.log("401 Unauthorized. Team has no access to requested domain")
return "401 Unauthorized"
return callback(**kwargs)
def get(mainOp, graph, op, *args, **qargs):
if mainOp != "graphs":
tangelo.http_status(400)
return {"error": "'%s' is not a legal method" % (mainOp)}
graph_path = graph.split(",")
if len(graph_path) == 2:
host = "localhost"
db = graph_path[0]
coll = graph_path[1]
elif len(graph_path) == 3:
host = graph_path[0]
db = graph_path[1]
coll = graph_path[2]
else:
tangelo.http_status(400)
return {"error": "Invalid graph name %s" % (graph)}
try:
m = MongoClient(host)[db][coll]
except ConnectionFailure:
tangelo.http_status(404)
return {"error": "Could not connect to graph %s" % (graph)}
if op == "vertices":
return vertices(m, *args, **qargs)
elif op == "edges":
return edges(m, *args, **qargs)
else:
tangelo.http_status(501)
return {"error": "Operation %s not implemented"}
def verifyDomainId(**kwargs):
if 'team_id' not in kwargs or 'domain_id' not in kwargs:
tangelo.http_status(500)
tangelo.log("team_id and domain_id required.")
return "team id and domain id required for this call."
team_id = int(kwargs['team_id'])
domain_id = int(kwargs['domain_id'])
if not datawake_mysql.hasDomains(team_id, domain_id):
tangelo.http_status(401)
tangelo.log(
"401 Unauthorized. Team has no access to requested domain")
return "401 Unauthorized"
return callback(**kwargs)
def verifyTeamId(**kwargs):
if 'team_id' not in kwargs:
tangelo.http_status(500)
tangelo.log("team_id required.")
return "team id required for this call."
user = get_user()
# verify the user can access the team
if not datawake_mysql.hasTeamAccess(user.get_email(),kwargs['team_id']):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log("401 Unauthorized. User has no access to requested team.")
return "401 Unauthorized"
return callback(**kwargs)
def copyToWebRoot(returnRec=None):
resultURI = returnRec['resultUri']
print 'copying pipelineURI:', resultURI
if resultURI is None:
tangelo.http_status(500)
return {'error': 'no resultURI for executed pipeline'}
if resultURI[0:7] == 'file://':
resultURI = resultURI[7:]
# copy the results file under the webroot so it can be read by
# javascript without having cross origin problems
shutil.copy(resultURI, 'pipelines')
print 'copy completed'
return resultURI
def stream_start(url, kwargs):
content = tangelo.server.analyze_url(url).content
if content is None or content.type != Content.Service:
tangelo.http_status(500, "Error Opening Streaming Service")
return {"error": "could not open streaming service"}
else:
# Extract the path to the service and the list of positional
# arguments.
module_path = content.path
pargs = content.pargs
# Get the service module.
try:
service = modules.get(module_path)
except:
tangelo.http_status(501, "Error Importing Streaming Service")
tangelo.content_type("application/json")
return tangelo.util.traceback_report(
error="Could not import module %s" % (module_path))
else:
# Check for a "stream" function inside the module.
if "stream" not in dir(service):
tangelo.http_status(400, "Non-Streaming Service")
return {
"error":
"The requested streaming service does not implement a 'stream()' function"
}
else:
# Call the stream function and capture its result.
try:
stream = service.stream(*pargs, **kwargs)
except Exception:
result = tangelo.util.traceback_report(
error=
"Caught exception during streaming service execution",
module=tangelo.request_path())
tangelo.log_warning(
"STREAM", "Could not execute service %s:\n%s" %
(tangelo.request_path(), "\n".join(
result["traceback"])))
tangelo.http_status(500,
"Streaming Service Raised Exception")
tangelo.content_type("application/json")
return result
else:
# Generate a key corresponding to this object.
key = tangelo.util.generate_key(streams)
# Log the object in the streaming table.
streams[key] = stream
# Create an object describing the logging of the generator object.
return {"key": key}
def getDataset(name):
global dataRoot
dataPath = os.path.abspath(os.path.join(dataRoot, name, 'data'))
datafile = os.path.join(dataPath, 'trainData.csv')
targetfile = os.path.join(dataPath, 'trainTargets.csv')
try:
reader = csv.reader(open(datafile))
except IOError:
tangelo.http_status(500)
return {'error': 'Could not open datafile for dataset %s' % (name)}
try:
treader = csv.reader(open(targetfile))
except IOError:
tangelo.http_status(500)
return {
'error': 'Could not open training data for dataset %s' % (name)
}
rows = list(reader)
trows = list(treader)
# read in the initial training data
dicts = []
for row in rows[1:]:
dicts.append({k: promote(v) for k, v in zip(rows[0], row)})
# read the training target variable
tdicts = []
for trow in trows[1:]:
tdicts.append({k: promote(v) for k, v in zip(trows[0], trow)})
trainName = tdicts[0].keys()[1]
# add the training variable to the datatable
for row in dicts:
indexToFix = row['d3mIndex']
row[trainName] = returnMatchTrain(indexToFix, tdicts, trainName)
schemaFile = os.path.join(dataPath, 'dataSchema.json')
schema = None
try:
with open(schemaFile) as f:
schema = json.loads(f.read())
except IOError:
tangelo.http_status(500)
return {'error': 'Could not open schemafile for dataset %s' % (name)}
except ValueError as e:
tangelo.http_status(500)
return {
'error':
'Error while parsing schemafile for dataset %s: %s' % (name, e)
}
return {'data': dicts, 'meta': schema, 'path': dataPath, 'name': name}
def edges(m, id="", key=None, value=None, **kwargs):
def convert(mongoEdge):
edge = {k: str(v) if isinstance(v, ObjectId) else v for k, v in mongoEdge.get("data", {}).iteritems()}
edge["_id"] = str(mongoEdge["_id"])
edge["_type"] = "edge"
edge["_inV"] = str(mongoEdge["source"])
edge["_outV"] = str(mongoEdge["target"])
return edge
if id == "" and key is None and value is None:
# Return all edges.
results = map(convert, m.find({"type": "link"}))
return {"version": "*.*", "results": results, "totalSize": len(results), "queryTime": 0.0}
tangelo.http_status(501)
return {"error": "unimplemented"}
def verifyTeamId(**kwargs):
if 'team_id' not in kwargs:
tangelo.http_status(500)
tangelo.log("team_id required.")
return "team id required for this call."
user = get_user()
# verify the user can access the team
if not datawake_mysql.hasTeamAccess(user.get_email(),
kwargs['team_id']):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log(
"401 Unauthorized. User has no access to requested team.")
return "401 Unauthorized"
return callback(**kwargs)
def forward(project, obj):
"""Forward an event object to the configured buildbot instance."""
auth = None
if projects.get('user') and projects.get('password'):
auth = (projects['user'], projects['password'])
resp = requests.post(
project['buildbot'].rstrip('/') + '/change_hook/github',
data={"payload": obj},
auth=auth
)
# headers={'CONTENT-TYPE': 'application/x-www-form-urlencoded'}
if resp.ok:
tangelo.http_status(200, 'OK')
return 'OK'
else:
tangelo.http_status(400, "Bad project configuration")
return 'Bad project configuration'
def get(team_id):
"""
Verify the logged in user has access to the requested team and return that teams list of domains.
:param team_id:
:return: List of domain objects for the team
[{'id':domain_id,'name','name':domainname,'description':domaindescription},..]
"""
user = helper.get_user()
if not db.hasTeamAccess(user.get_email(),team_id):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log("401 Unauthorized domains/?team_id="+str(team_id)+" user: "******"401 Unauthorized"
else:
results = db.get_domains(team_id)
return json.dumps(results)
def get(team_id):
"""
Verify the logged in user has access to the requested team and return that teams list of domains.
:param team_id:
:return: List of domain objects for the team
[{'id':domain_id,'name','name':domainname,'description':domaindescription},..]
"""
user = helper.get_user()
if not db.hasTeamAccess(user.get_email(), team_id):
tangelo.content_type()
tangelo.http_status(401)
tangelo.log("401 Unauthorized domains/?team_id=" + str(team_id) +
" user: "******"401 Unauthorized"
else:
results = db.get_domains(team_id)
return json.dumps(results)
def add_trail(team_id,domain_id,name,description=''):
tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id))
#check the trail name
if name is None or len(name) < 1:
raise ValueError("Trail names must have at least one character")
user = helper.get_user()
# create and then return the new trail
try:
newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email())
except Exception as e:
tangelo.log(e)
tangelo.http_status(501)
return "Failed to create trail. This trail name may already be used."
newTrail = dict(id=newTrailId,name=name,description=description)
tangelo.log(newTrail)
return json.dumps(newTrail)
def add_trail(team_id,domain_id,name,description=''):
tangelo.log('datawake_trails POST name=%s description=%s domain=%s team=%s' % (name, description, domain_id,team_id))
#check the trail name
if name is None or len(name) < 1:
raise ValueError("Trail names must have at least one character")
user = helper.get_user()
# create and then return the new trail
try:
newTrailId = db.addTrail(team_id,domain_id,name, description, user.get_email())
newTrail = dict(id=newTrailId,name=name,description=description)
tangelo.log(newTrail)
return json.dumps(newTrail)
except Exception as e:
tangelo.log(e)
tangelo.http_status(501)
return "Failed to create trail. This trail name may already be used."
def stream_next(key):
if key not in streams:
tangelo.http_status(404, "No Such Key")
return {"error": "Stream key does not correspond to an active stream",
"stream": key}
else:
# Grab the stream in preparation for running it.
stream = streams[key]
# Attempt to run the stream via its next() method - if this
# yields a result, then continue; if the next() method raises
# StopIteration, then there are no more results to retrieve; if
# any other exception is raised, this is treated as an error.
try:
return stream.next()
except StopIteration:
del streams[key]
tangelo.http_status(204, "Stream Finished")
return "OK"
except:
del streams[key]
tangelo.http_status(500, "Streaming Service Exception")
tangelo.content_type("application/json")
error_code = tangelo.util.generate_error_code()
tangelo.util.log_traceback("STREAM", error_code, "Offending stream key: %s" % (key), "Uncaught executing executing service %s" % (tangelo.request_path))
return tangelo.util.error_report(error_code)
def stream_next(key):
if key not in streams:
tangelo.http_status(404, "No Such Key")
return {
"error": "Stream key does not correspond to an active stream",
"stream": key
}
else:
# Grab the stream in preparation for running it.
stream = streams[key]
# Attempt to run the stream via its next() method - if this
# yields a result, then continue; if the next() method raises
# StopIteration, then there are no more results to retrieve; if
# any other exception is raised, this is treated as an error.
try:
return stream.next()
except StopIteration:
del streams[key]
tangelo.http_status(204, "Stream Finished")
return "OK"
except:
del streams[key]
tangelo.http_status(500, "Streaming Service Exception")
tangelo.content_type("application/json")
return tangelo.util.traceback_report(
error="Caught exception while executing stream service",
stream=key)
def stream_start(url, kwargs):
content = tangelo.server.analyze_url(url).content
if content is None or content.type != Content.Service:
tangelo.http_status(500, "Error Opening Streaming Service")
return {"error": "could not open streaming service"}
else:
# Extract the path to the service and the list of positional
# arguments.
module_path = content.path
pargs = content.pargs
# Get the service module.
try:
service = modules.get(module_path)
except:
tangelo.http_status(500, "Error Importing Streaming Service")
tangelo.content_type("application/json")
error_code = tangelo.util.generate_error_code()
tangelo.util.log_traceback("STREAM", error_code, "Could not import module %s" % (tangelo.request_path()))
return tangelo.util.error_report(error_code)
else:
# Check for a "stream" function inside the module.
if "stream" not in dir(service):
tangelo.http_status(400, "Non-Streaming Service")
return {"error": "The requested streaming service does not implement a 'stream()' function"}
else:
# Call the stream function and capture its result.
try:
stream = service.stream(*pargs, **kwargs)
except Exception:
tangelo.http_status(500, "Streaming Service Raised Exception")
tangelo.content_type("application/json")
error_code = tangelo.util.generate_error_code()
tangelo.util.log_traceback("STREAM", error_code, "Could not execute service %s" % (tangelo.request_path()))
return tangelo.util.error_report(error_code)
else:
# Generate a key corresponding to this object.
key = tangelo.util.generate_key(streams)
# Log the object in the streaming table.
streams[key] = stream
# Create an object describing the logging of the generator object.
return {"key": key}
def post(port=None):
# get the protocol version
version = core_pb2.DESCRIPTOR.GetOptions().Extensions[
core_pb2.protocol_version]
# get the address from an environment variable. This must be set in the executing shell.
# During automated evaluation runs, the environment variable will be set by Kubernetes
server_channel_address = os.environ.get('TA2_SERVER_CONN')
# complain in the return if we didn't get an address to connect to
if server_channel_address is None:
tangelo.http_status(500)
return {'error': 'TA2_SERVER_CONN environment variable is not set!'}
#channel = grpc.insecure_channel('localhost:%d' % (int(port)))
channel = grpc.insecure_channel(server_channel_address)
stub = core_pb2_grpc.CoreStub(channel)
resp = stub.StartSession(
core_pb2.SessionRequest(user_agent='modsquad', version=version))
return MessageToJson(resp)
def vertices(m, id="", op=None, key=None, value=None, **kwargs):
if id != "":
# Return the requested vertex
vertex = m.find_one({"_id": ObjectId(id)})
if vertex is None or vertex["type"] != "node":
tangelo.http_status(404)
return {"error": "No such vertex"}
response = {
"version": "*.*",
"results": {
"_type": "vertex",
"_id": id
},
"queryTime": 0.0
}
for key, value in vertex.get("data", {}).iteritems():
response["results"][key] = value
return response
else:
def process(rec):
result = {"_type": "vertex", "_id": str(rec["_id"])}
result.update({k: v for k, v in rec.get("data", {}).iteritems()})
return result
vertices = map(process, m.find({"type": "node"}))
response = {
"version": "*.*",
"results": vertices,
"totalSize": len(vertices),
"queryTime": 0.0
}
return response
def run(method=None, data=None, **kwargs):
if data is None:
tangelo.http_status(400, 'Required argument missing')
return None
fn = select_method(method)
if fn is None:
tangelo.http_status(400, 'Required argument missing')
return None
data = json.loads(data)
dataframe = make_frame(data)
if fn == run_quadratic:
predictor_variables = kwargs.get('predictor_variables')
response = kwargs.get('response')
quadratic_variables = kwargs.get('quadratic_variables')
if predictor_variables is None or response is None or quadratic_variables is None:
tangelo.http_status(400, 'Required argument missing')
return None
args = map(json.loads,
[response, predictor_variables, quadratic_variables])
elif fn in [run_lm, run_loess]:
predictor_variables = kwargs.get('predictor_variables')
response = kwargs.get('response')
if predictor_variables is None or response is None:
tangelo.http_status(400, 'Required argument missing')
return None
args = map(json.loads, [response, predictor_variables])
else:
tangelo.http_status(500, 'Impossible condition')
return str(fn(dataframe, *args))
def post(*arg, **kwarg):
"""Listen for github webhooks, authenticate, and forward to buildbot."""
# retrieve the headers from the request
try:
received = tangelo.request_header('X-Hub-Signature')[5:]
except Exception:
received = ''
# get the request body as a dict
# for json
body = tangelo.request_body().read()
try:
obj = json.loads(body)
except:
tangelo.http_status(400, "Could not load json object")
return "Could not load json object"
# obj = json.loads(kwarg['payload'])
open('last.json', 'w').write(json.dumps(obj, indent=2))
project = get_project(obj.get('repository', {}).get('full_name'))
if project is None:
tangelo.http_status(400, "Unknown project")
return 'Unknown project'
# make sure this is a valid request coming from github
if not authenticate(project.get('api-key', ''), body, received):
tangelo.http_status(403, "Invalid signature")
return 'Invalid signature'
event = tangelo.request_header('X-Github-Event')
if project['events'] == '*' or event in project['events']:
obj['event'] = event
# add a new item to the test queue
return forward(project, body)
else:
tangelo.http_status(200, "Unhandled event")
return 'Unhandled event'
def run():
config_file = os.environ.get('JSON_CONFIG_PATH')
print 'environment variable said:', config_file
#config_file = "/Users/clisle/proj/D3M/code/eval/config.json"
print 'config service: looking for config file..', config_file
if config_file is None:
tangelo.http_status(500)
return {'error': 'JSON_CONFIG_PATH is not set!'}
try:
with open(config_file) as f:
text = f.read()
except IOError as e:
tangelo.http_status(500)
return {'error': str(e)}
try:
config = json.loads(text)
except ValueError as e:
tangelo.http_status(500)
return {'error': 'Could not parse JSON - %s' % (str(e))}
# make place for executed pipelines to be copied (see pipeline.py copyToWebRoot).
# ignore exceptions because directory might exist if rerun on new problems without
# resetting (should only occur on native execution instead of single problem container execution)
try:
os.mkdir('pipelines')
except:
pass
print 'received json configuration:', config
os.environ['PROBLEM_SCHEMA_PATH'] = config['problem_schema']
os.environ['DATASET_SCHEMA_PATH'] = config['dataset_schema']
os.environ['TRAINING_DATA_ROOT'] = config['training_data_root']
os.environ['PROBLEM_ROOT'] = config['problem_root']
os.environ['EXECUTABLES_ROOT'] = config['executables_root']
# used by the ta2read service
os.environ['TEMP_STORAGE_ROOT'] = config['temp_storage_root']
return config
def post(*pathcomp, **kwargs):
if len(pathcomp) == 0:
# TODO: raise error condition
pass
action = pathcomp[0]
args = pathcomp[1:]
if action == "start":
if len(args) == 0:
tangelo.http_status(400, "Path To Service Required")
return {"error": "No service path was specified"}
return stream_start("/" + "/".join(args), kwargs)
elif action == "next":
if len(args) != 1:
tangelo.http_status(400, "Stream Key Required")
return {"error": "No stream key was specified"}
return stream_next(args[0])
else:
tangelo.http_status(400, "Illegal POST action")
return {"error": "Illegal POST action '%s'" % (action)}
def run():
config_file = os.environ.get('JSON_CONFIG_PATH')
if config_file is None:
tangelo.http_status(500)
return {'error': 'JSON_CONFIG_PATH is not set!'}
try:
with open(config_file) as f:
text = f.read()
except IOError as e:
tangelo.http_status(500)
return {'error': str(e)}
try:
config = json.loads(text)
except ValueError as e:
tangelo.http_status(500)
return {'error': 'Could not parse JSON - %s' % (str(e))}
os.environ['PROBLEM_SCHEMA'] = config['problem_schema']
os.environ['DATASET_SCHEMA'] = config['dataset_schema']
os.environ['TRAINING_DATA_ROOT'] = config['training_data_root']
return config
def run(plot_type, sort=False):
if plot_type == "count":
return count_barchart(sort=sort)
else:
tangelo.http_status(400, "Illegal plot type")
return {"error": "Illegal plot type: %s" % (plot_type)}
def run():
tangelo.http_status(400, "Bad Configuration")
return {"error": msg}
def post(*pargs, **query):
args = query.get("args", "")
timeout = float(query.get("timeout", 0))
if len(pargs) == 0:
tangelo.http_status(400, "Required Argument Missing")
return {"error": "No program path was specified"}
program_url = "/" + "/".join(pargs)
content = analyze_url(program_url).content
if content is None or content.type != Content.File:
tangelo.http_status(404, "Not Found")
return {"error": "Could not find a script at %s" % (program_url)}
program = content.path
# Check the user arguments.
userargs = args.split()
if "--port" in userargs:
tangelo.http_status(400, "Illegal Argument")
return {"error": "You may not specify '--port' among the arguments passed in 'args'"}
# Obtain an available port.
port = tangelo.util.get_free_port()
# Generate a unique key.
key = tangelo.util.generate_key(processes.keys())
# Detect http vs. https
scheme = "ws"
ssl_key = cherrypy.config.get("server.ssl_private_key")
ssl_cert = cherrypy.config.get("server.ssl_certificate")
# Generate command line.
cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs
if ssl_key and ssl_cert:
scheme = "wss"
cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert])
# Launch the requested process.
tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline)))
try:
process = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except (OSError, IOError) as e:
tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process")
return {"error": e.strerror}
# Capture the new process's stdout and stderr streams in
# non-blocking readers.
stdout = tangelo.util.NonBlockingReader(process.stdout)
stderr = tangelo.util.NonBlockingReader(process.stderr)
# Read from stdout to look for the signal that the process has
# started properly.
class FactoryStarted:
pass
class Failed:
pass
class Timeout:
pass
signal = "Starting factory"
if timeout <= 0:
timeout = 10
sleeptime = 0.5
wait = 0
saved_lines = []
try:
while True:
lines = stdout.readlines()
saved_lines += lines
for line in lines:
if line == "":
# This indicates that stdout has closed without
# starting the process.
raise Failed()
elif signal in line:
# This means that the server has started.
raise FactoryStarted()
# If neither failure nor success occurred in the last block
# of lines from stdout, either time out, or try again after
# a short delay.
if wait >= timeout:
raise Timeout()
wait += sleeptime
time.sleep(sleeptime)
except Timeout:
tangelo.http_status(524, "Timeout")
return {"error": "Process startup timed out"}
except Failed:
tangelo.http_status(500)
return {"error": "Process did not start up properly",
"stdout": saved_lines,
"stderr": stderr.readlines()}
except FactoryStarted:
stdout.pushlines(saved_lines)
# Create a websocket handler path dedicated to this process.
host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host
tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp")
# Log the new process in the process table, including non-blocking
# stdout and stderr readers.
processes[key] = {"port": port,
"process": process,
"stdout": stdout,
"stderr": stderr}
# Form the websocket URL from the hostname/port used in the
# request, and the newly generated key.
url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key)
return {"key": key,
"url": url}
def has_session(**kwargs):
if 'user' in cherrypy.session:
return callback(**kwargs)
tangelo.http_status(401)
tangelo.log("401 Unauthorized No user in session")
return "No User in the current session"
def get_stream_info(key):
tangelo.http_status(501)
return {"error": "stream info method currently unimplemented"}
def post(*arg, **kwarg):
"""Listen for github webhooks, authenticate, and forward to buildbot."""
# retrieve the headers from the request
print "MASTER RECEIVED A POST EVENT"
# print "TGELO CONFI",tangelo.cherrypy.request.header_list
try:
received = tangelo.request_header('X-Hub-Signature')[5:]
except Exception:
try:
received = tangelo.request_header('BOT-Signature')[5:]
except Exception:
received = ''
# get the request body as a dict
# for json
body = tangelo.request_body().read()
try:
obj = json.loads(body)
except:
tangelo.http_status(400, "Could not load json object")
return "Could not load json object"
# obj = json.loads(kwarg['payload'])
#open('last.json', 'w').write(json.dumps(obj, indent=2))
project_name = obj.get('repository', {}).get('full_name')
project = get_project(project_name)
if project is None:
tangelo.http_status(400, "Unknown project")
return 'Unknown project'
# make sure this is a valid request coming from github
if not authenticate(project.get('api-key', ''), body, received) \
and \
not authenticate(project.get('bot-key', ''), body, received):
tangelo.http_status(403, "Invalid signature")
return 'Invalid signature'
event = tangelo.request_header('X-Github-Event')
print "EVENT:",event
if project['github-events'] == '*' or event in project['github-events']:
obj['event'] = event
try:
commit = obj["head_commit"]
is_commit = True
except:
commit = obj["pull_request"]
is_commit = False
try:
act = obj["action"]
if act == "closed": # closed PR no need to run anything
return "Closed PR, skipping"
except:
pass
if commit is None:
## no head_Commit trying to see if it's a pull request
return "Null Head Commit Found, not a PR either skipping"
if is_commit:
commit_id = commit["id"]
commit_msg = commit["message"]
else:
## It 's a PR faking the head_commit/id bits for slaves
commits_url = commit["commits_url"]
commit_id = commit["head"]["sha"]
commit_statuses_url=commit["statuses_url"]
commit_ref = commit["head"]["ref"]
resp = requests.get(commits_url,verify=False)
commit = resp.json()[-1]["commit"]
commit_msg=commit["message"]
commit["id"]=commit_id
obj["ref"]=commit_ref
commit["statuses_url"]=commit_statuses_url
obj["head_commit"]=commit
signature = hmac.new(str(project["bot-key"]), json.dumps(obj), hashlib.sha1).hexdigest()
if commit_msg.find("##bot##skip-commit")>-1:
# User requested to not send this commit to bots
return "Skipped testing commit '%s' at committer request (found string '##bot##skip-commit')"
nok = 0
for islave, slave in enumerate(project["slaves"]):
islaves = commit_msg.find("##bot##skip-slaves")
if islaves>-1:
# ok some slaves need skipping
msg = commit_msg[islaves+18:]
iend = msg.find("\n")
msg = msg[:iend].strip().split()
iskip = False
for m in msg:
if slave.find(m)>-1:
iskip = True
break
if iskip:
print "\033[%im" % (91+islave),"Commit asked to skip:",slave,"\033[0m"
nok+=1
continue
print "\033[%im" % (91+islave),"SENDING TO:",slave,"\033[0m"
try:
resp = forward(slave,obj,signature)
if resp.ok:
nok+=1
except:
print "\033[%im" % (91+islave),"could not connect","\033[0m"
nok+=1
if nok>0:
return "Ok sent this to %i slaves out of %i" % (nok,len(project["slaves"]))
else:
msg = "All slaves failed to respond, last error was: %s" % resp.text
print msg
tangelo.http_status(resp.status_code, msg)
return msg
elif tangelo.request_header('BOT-Event') == "status":
## put here code to update status of commit on github
headers = {
"Authorization":"token "+project["token"],
}
commit_id = obj["commit"]["id"]
if obj["code"] == 0:
state = "success"
elif obj["code"] is None:
state = "pending"
else:
state = "failure"
slave = obj["slave_host"]
try:
islave = project["slaves"].find("http://"+slave)
except:
islave = -91 # Turn off styling
pth = os.path.join(project["logs_dir"],slave,project_name,commit_id)
print "\033[%im" % (91+islave),"DUMPING INFO IN:",pth,"\033[0m"
print "\033[%im" % (91+islave),"could not connect","\033[0m"
if not os.path.exists(str(pth)):
print "Creating:",pth
os.makedirs(pth)
f=open(os.path.join(pth,cmd2str(obj["command"])),"w")
print >>f,"<html><body>"
print >>f,"<h1>%s (%s)</h1><br><h2>commit: %s<h2>" % (project_name,obj["slave_name"],commit_id)
host = tangelo.cherrypy.url()
host=host[host.find("//")+2:]
if obj["previous"] is not None:
ptarget = "http://%s/%s/%s/%s/%s" % (host,slave,project_name,commit_id,cmd2str(obj["previous"]))
print >>f, "<h2>PREVIOUS COMMAND</h2>"
print >>f,"<a href='%s'>" % ptarget,obj["previous"],"</a>"
print >>f, "<h2>COMMAND</h2>"
print >>f,"<pre>",obj["command"],"</pre>"
if obj["command"].find("ctest")>-1:
print >>f, "<h3>CTEST PAGE</h3>"
build_name = "%s-%s" % (slave.replace(":",""),commit_id)
ptarget = "https://open.cdash.org/index.php?compare1=65&filtercount=2&field1=buildname%%2Fstring&project=UV-CDAT&field2=buildstarttime%%2Fdate&value1=%s" % build_name
print >>f,"<A HREF='%s'>Click here</A>" % ptarget
print >>f, "<h3>OUTPUT</h3>"
print >>f,"<pre>",obj["output"],"</pre>"
print >>f, "<h3>ERROR</h3>"
print >>f,"<pre>",obj["error"],"</pre>"
print >>f,"</body></html>"
f.close()
target = "http://%s/%s/%s/%s/%s" % (host,slave,project_name,commit_id,cmd2str(obj["command"]))
context = "cont-int/LLNL/%s-%s" % (obj["os"],obj["slave_name"])
data = {
"state":state,
"target_url": target,
"description": "'%s' (%s)" % (obj["command"][:20],time.asctime()),
"context": context,
}
resp = requests.post(
obj["commit"]["statuses_url"].replace("{sha}",obj["commit"]["id"]),
data = json.dumps(data),
verify = False,
headers = headers)
return "Received and treated a BOT STATUS update event"
else:
tangelo.http_status(200, "Unhandled event")
return 'Unhandled event'
def invoke_service(self, module, *pargs, **kwargs):
# TODO(choudhury): This method should attempt to load the named module,
# then invoke it with the given arguments. However, if the named
# module is "config" or something similar, the method should instead
# launch a special "config" app, which lists the available app modules,
# along with docstrings or similar. It should also allow the user to
# add/delete search paths for other modules.
tangelo.content_type("text/plain")
# Save the system path (be sure to *make a copy* using the list()
# function) - it will be modified before invoking the service, and must
# be restored afterwards.
origpath = list(sys.path)
# By default, the result should be an object with error message in if
# something goes wrong; if nothing goes wrong this will be replaced
# with some other object.
result = {}
# Store the modpath in the thread-local storage (tangelo.paths() makes
# use of this per-thread data, so this is the way to get the data
# across the "module boundary" properly).
modpath = os.path.dirname(module)
cherrypy.thread_data.modulepath = modpath
cherrypy.thread_data.modulename = module
# Extend the system path with the module's home path.
sys.path.insert(0, modpath)
try:
service = self.modules.get(module)
except:
tangelo.http_status(501, "Error Importing Service")
tangelo.content_type("application/json")
result = {"error": "Could not import module %s" % (tangelo.request_path()),
"traceback": traceback.format_exc().split("\n")}
else:
# Try to run the service - either it's in a function called
# "run()", or else it's in a REST API consisting of at least one of
# "get()", "put()", "post()", or "delete()".
#
# Collect the result in a variable - depending on its type, it will
# be transformed in some way below (by default, to JSON, but may
# also raise a cherrypy exception, log itself in a streaming table,
# etc.).
try:
if 'run' in dir(service):
# Call the module's run() method, passing it the positional
# and keyword args that came into this method.
result = service.run(*pargs, **kwargs)
else:
# Reaching here means it's a REST API. Check for the
# requested method, ensure that it was marked as being part
# of the API, and call it; or give a 405 error.
method = cherrypy.request.method
restfunc = service.__dict__.get(method.lower())
if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful):
result = restfunc(*pargs, **kwargs)
else:
tangelo.http_status(405, "Method Not Allowed")
tangelo.content_type("application/json")
result = {"error": "Method '%s' is not allowed in this service" % (method)}
except:
stacktrace = traceback.format_exc()
tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), stacktrace))
tangelo.http_status(501, "Web Service Error")
tangelo.content_type("application/json")
result = {"error": "Error executing service",
"module": tangelo.request_path(),
"traceback": stacktrace.split("\n")}
# Restore the path to what it was originally.
sys.path = origpath
# If the result is not a string, attempt to convert it to one via JSON
# serialization. This allows services to return a Python object if they
# wish, or to perform custom serialization (such as for MongoDB results,
# etc.).
if not isinstance(result, types.StringTypes):
try:
result = json.dumps(result)
except TypeError as e:
tangelo.http_status(400, "JSON Error")
tangelo.content_type("application/json")
result = {"error": "JSON type error executing service",
"message": e.message}
else:
tangelo.content_type("application/json")
return result
def invoke_service(self, module, *pargs, **kwargs):
tangelo.content_type("text/plain")
# Save the system path (be sure to *make a copy* using the list()
# function). This will be restored to undo any modification of the path
# done by the service.
origpath = list(sys.path)
# By default, the result should be an object with error message in if
# something goes wrong; if nothing goes wrong this will be replaced
# with some other object.
result = {}
# Store the modpath in the thread-local storage (tangelo.paths() makes
# use of this per-thread data, so this is the way to get the data
# across the "module boundary" properly).
modpath = os.path.dirname(module)
cherrypy.thread_data.modulepath = modpath
cherrypy.thread_data.modulename = module
# Change the current working directory to that of the service module,
# saving the old one. This is so that the service function executes as
# though it were a Python program invoked normally, and Tangelo can
# continue running later in whatever its original CWD was.
save_cwd = os.getcwd()
os.chdir(modpath)
try:
service = self.modules.get(module)
except:
tangelo.http_status(501, "Error Importing Service")
tangelo.content_type("application/json")
result = tangelo.util.traceback_report(error="Could not import module %s" % (tangelo.request_path()))
else:
# Try to run the service - either it's in a function called
# "run()", or else it's in a REST API consisting of at least one of
# "get()", "put()", "post()", or "delete()".
#
# Collect the result in a variable - depending on its type, it will
# be transformed in some way below (by default, to JSON, but may
# also raise a cherrypy exception, log itself in a streaming table,
# etc.).
try:
if "run" in dir(service):
# Call the module's run() method, passing it the positional
# and keyword args that came into this method.
result = service.run(*pargs, **kwargs)
else:
# Reaching here means it's a REST API. Check for the
# requested method, ensure that it was marked as being part
# of the API, and call it; or give a 405 error.
method = cherrypy.request.method
restfunc = service.__dict__.get(method.lower())
if (restfunc is not None and hasattr(restfunc, "restful") and restfunc.restful):
result = restfunc(*pargs, **kwargs)
else:
tangelo.http_status(405, "Method Not Allowed")
tangelo.content_type("application/json")
result = {"error": "Method '%s' is not allowed in this service" % (method)}
except:
tangelo.http_status(501, "Web Service Error")
tangelo.content_type("application/json")
result = tangelo.util.traceback_report(error="Error executing service", module=tangelo.request_path())
tangelo.log_warning("SERVICE", "Could not execute service %s:\n%s" % (tangelo.request_path(), "\n".join(result["traceback"])))
# Restore the path to what it was originally.
sys.path = origpath
# Restore the CWD to what it was before the service invocation.
os.chdir(save_cwd)
# If the result is not a string, attempt to convert it to one via JSON
# serialization. This allows services to return a Python object if they
# wish, or to perform custom serialization (such as for MongoDB results,
# etc.).
if not isinstance(result, types.StringTypes):
try:
result = json.dumps(result)
except TypeError as e:
tangelo.http_status(400, "JSON Error")
tangelo.content_type("application/json")
result = {"error": "JSON type error executing service",
"message": e.message}
else:
tangelo.content_type("application/json")
return result
def post(*pargs, **query):
args = query.get("args", "")
timeout = float(query.get("timeout", 0))
processes = tangelo.plugin_store()["processes"]
if len(pargs) == 0:
tangelo.http_status(400, "Required Argument Missing")
return {"error": "No program path was specified"}
program_url = "/" + "/".join(pargs)
content = analyze_url(program_url).content
if content is None or content.type != Content.File:
tangelo.http_status(404, "Not Found")
return {"error": "Could not find a script at %s" % (program_url)}
elif content.path is None:
tangelo.http_status(403, "Restricted")
return {"error": "The script at %s is access-restricted"}
program = content.path
# Check the user arguments.
userargs = args.split()
if "--port" in userargs:
tangelo.http_status(400, "Illegal Argument")
return {"error": "You may not specify '--port' among the arguments passed in 'args'"}
# Obtain an available port.
port = tangelo.util.get_free_port()
# Generate a unique key.
key = tangelo.util.generate_key(processes.keys())
# Detect http vs. https
scheme = "ws"
ssl_key = cherrypy.config.get("server.ssl_private_key")
ssl_cert = cherrypy.config.get("server.ssl_certificate")
# Generate command line.
cmdline = [vtkpython, weblauncher, program, "--port", str(port)] + userargs
if ssl_key and ssl_cert:
scheme = "wss"
cmdline.extend(["--sslKey", ssl_key, "--sslCert", ssl_cert])
# Launch the requested process.
tangelo.log_info("VTKWEB", "Starting process: %s" % (" ".join(cmdline)))
try:
process = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except (OSError, IOError) as e:
tangelo.log_warning("VTKWEB", "Error: could not launch VTKWeb process")
return {"error": e.strerror}
# Capture the new process's stdout and stderr streams in
# non-blocking readers.
stdout = tangelo.util.NonBlockingReader(process.stdout)
stderr = tangelo.util.NonBlockingReader(process.stderr)
# Read from stdout to look for the signal that the process has
# started properly.
class FactoryStarted:
pass
class Failed:
pass
class Timeout:
pass
signal = "Starting factory"
if timeout <= 0:
timeout = 10
sleeptime = 0.5
wait = 0
saved_lines = []
try:
while True:
lines = stdout.readlines()
saved_lines += lines
for line in lines:
if line == "":
# This indicates that stdout has closed without
# starting the process.
raise Failed()
elif signal in line:
# This means that the server has started.
raise FactoryStarted()
# If neither failure nor success occurred in the last block
# of lines from stdout, either time out, or try again after
# a short delay.
if wait >= timeout:
raise Timeout()
wait += sleeptime
time.sleep(sleeptime)
except Timeout:
tangelo.http_status(524, "Timeout")
return {"error": "Process startup timed out"}
except Failed:
tangelo.http_status(500)
return {"error": "Process did not start up properly",
"stdout": saved_lines,
"stderr": stderr.readlines()}
except FactoryStarted:
stdout.pushlines(saved_lines)
# Create a websocket handler path dedicated to this process.
host = "localhost" if cherrypy.server.socket_host == "0.0.0.0" else cherrypy.server.socket_host
tangelo.websocket.mount(key, WebSocketRelay(host, port, key), "wamp")
# Log the new process in the process table, including non-blocking
# stdout and stderr readers.
processes[key] = {"port": port,
"process": process,
"stdout": stdout,
"stderr": stderr}
# Form the websocket URL from the hostname/port used in the
# request, and the newly generated key.
url = "%s://%s/ws/%s/ws" % (scheme, cherrypy.request.base.split("//")[1], key)
return {"key": key,
"url": url}
def run():
tangelo.http_status(400, "Bad Configuration")
return {"error": msg}
