def mock_myria_get(url, request):
query_params = urlparse.parse_qs(url.query)
# The below JSON responses are taken directly from production Myria instance
# with vega.cs.washington.edu:1776 changed to fakefake:12345
queries = [{"url":"http://fake.fake:12345/query/query-140", "queryId":140, "rawQuery":"JustX(x) :- TwitterK(x,y)", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-26T15:10:37.718-08:00", "startTime":"2014-02-26T15:10:37.878-08:00", "finishTime":"2014-02-26T15:10:38.648-08:00", "message":None, "elapsedNanos":769418780, "status":"SUCCESS"}, {"url":"http://fake.fake:12345/query/query-139", "queryId":139, "rawQuery":"download [public#adhoc#TwitterK]", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-26T00:51:54.868-08:00", "startTime":"2014-02-26T00:51:54.963-08:00", "finishTime":"2014-02-26T00:51:55.002-08:00", "message":None, "elapsedNanos":38598641, "status":"SUCCESS"}, {"url":"http://fake.fake:12345/query/query-138", "queryId":138, "rawQuery":"download [public#adhoc#Twitter]", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-26T00:36:54.845-08:00", "startTime":"2014-02-26T00:36:54.970-08:00", "finishTime":"2014-02-26T00:36:55.222-08:00", "message":None, "elapsedNanos":252420874, "status":"KILLED"}, {"url":"http://fake.fake:12345/query/query-137", "queryId":137, "rawQuery":"download [public#adhoc#TwitterK]", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-26T00:10:06.066-08:00", "startTime":"2014-02-26T00:10:06.173-08:00", "finishTime":"2014-02-26T00:10:06.926-08:00", "message":None, "elapsedNanos":752888288, "status":"SUCCESS"}, {"url":"http://fake.fake:12345/query/query-136", "queryId":136, "rawQuery":"download [public#adhoc#TwitterK]", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T23:59:33.023-08:00", "startTime":"2014-02-25T23:59:33.181-08:00", "finishTime":"2014-02-25T23:59:33.337-08:00", "message":None, "elapsedNanos":155921007, "status":"KILLED"}, {"url":"http://fake.fake:12345/query/query-135", "queryId":135, "rawQuery":"download [public#__TEMP__#JustX]", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T23:55:20.921-08:00", "startTime":"2014-02-25T23:55:21.126-08:00", "finishTime":"2014-02-25T23:55:21.396-08:00", "message":None, "elapsedNanos":270059319, "status":"KILLED"}, {"url":"http://fake.fake:12345/query/query-134", "queryId":134, "rawQuery":"JustX(x) :- TwitterK(x,y)", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T23:07:21.028-08:00", "startTime":"2014-02-25T23:07:21.469-08:00", "finishTime":"2014-02-25T23:07:22.765-08:00", "message":None, "elapsedNanos":1295696742, "status":"SUCCESS"}, {"url":"http://fake.fake:12345/query/query-133", "queryId":133, "rawQuery":"JustX(x) :- TwitterK(x,y)", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T18:01:57.636-08:00", "startTime":None, "finishTime":None, "message":None, "elapsedNanos":None, "status":"UNKNOWN"}, {"url":"http://fake.fake:12345/query/query-132", "queryId":132, "rawQuery":"all_opp_vct =\nSELECT Opp.*, Vct.pop, Vct.support\nFROM SCAN(armbrustlab:seaflow:all_opp_v3) AS Opp,\n SCAN(armbrustlab:seaflow:all_vct) AS Vct\nWHERE (Opp.Cruise = Vct.Cruise)\n AND (Opp.Day = Vct.Day)\n AND (Opp.File_Id = Vct.File_Id)\n AND (Opp.Cell_Id = Vct.Cell_Id);", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T17:05:18.617-08:00", "startTime":None, "finishTime":None, "message":None, "elapsedNanos":None, "status":"UNKNOWN"}, {"url":"http://fake.fake:12345/query/query-131", "queryId":131, "rawQuery":"T1 = SCAN(TwitterK);\n\nT2 = [FROM T1 EMIT $0 AS x];\n\nSTORE (T2, JustX);", "logicalRa":None, "physicalPlan":None, "submitTime":"2014-02-25T15:28:47.782-08:00", "startTime":"2014-02-25T15:28:48.668-08:00", "finishTime":"2014-02-25T15:28:50.177-08:00", "message":None, "elapsedNanos":1509299867, "status":"SUCCESS"}]
twitter_dataset = {"relationKey":{"userName":"******", "programName":"adhoc", "relationName":"Twitter"}, "schema":{"columnTypes":["INT_TYPE", "INT_TYPE"], "columnNames":["followee", "follower"]}, "numTuples":-1, "queryId":2, "created":"2014-02-09T12:40:43.438-08:00", "uri":"http://fake.fake:12345/dataset/user-public/program-adhoc/relation-Twitter"}
if url.path == '/workers':
return jstr({'1': 'localhost:12347', '2': 'localhost:12348'})
elif url.path == '/workers/alive':
return jstr([1, 2])
elif url.path == '/dataset':
return jstr([twitter_dataset])
elif url.path == '/dataset/user-public/program-adhoc/relation-Twitter':
return jstr(twitter_dataset)
elif url.path == '/query':
headers = {
'X-Count': len(queries)}
limit = int((query_params.get('limit') or [10])[0])
return {'status_code': 200, 'content': jstr(queries[:limit]), 'headers': headers}
elif url.path == '/query/query-140':
return {'status_code': 201,
'headers': {'Location': 'http://fake.fake:12345/query/query-140'},
'content': queries[0]}
print >> sys.stderr, "Did not handle URL {}".format(url)
def local_mock(url, request):
global query_counter
if url.path == '/workers':
return jstr({'1': 'localhost:12347', '2': 'localhost:12348'})
elif url.path == '/workers/alive':
return jstr([1, 2])
elif url.path == '/workers/worker-1':
return jstr("localhost:12347")
return None
def local_mock(url, request):
global query_counter
if url.path == '/workers':
return jstr({'1': 'localhost:12347', '2': 'localhost:12348'})
elif url.path == '/workers/alive':
return jstr([1, 2])
elif url.path == '/workers/worker-1':
return jstr("localhost:12347")
return None
def mock_TwitterK(url, request):
if url.path == '/dataset':
fields = dict(request.body.fields)
relation_key = get_field(fields, 'relationKey')
schema = get_field(fields, 'schema')
overwrite = get_field(fields, 'overwrite')
if not overwrite:
return {'status_code': 409,
'content': 'That dataset already exists.'}
eq_(relation_key['userName'], 'test')
eq_(relation_key['programName'], 'testp')
if relation_key['relationName'] == 'TwitterK':
eq_(schema['columnNames'], ['src', 'dst'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'LONG_TYPE'])
elif relation_key['relationName'] == 'TwitterKnoheader':
eq_(schema['columnNames'], ['column0', 'column1'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'LONG_TYPE'])
elif relation_key['relationName'] == 'plaintext':
eq_(schema['columnNames'], ['number', 'value'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'STRING_TYPE'])
elif relation_key['relationName'] == 'float':
eq_(schema['columnNames'], ['field1', 'field2'])
eq_(schema['columnTypes'], ['DOUBLE_TYPE', 'DOUBLE_TYPE'])
elif relation_key['relationName'] == 'nulls':
eq_(schema['columnNames'], ['field1', 'field2', 'field3'])
eq_(schema['columnTypes'],
['LONG_TYPE', 'STRING_TYPE', 'STRING_TYPE'])
elif relation_key['relationName'] in ['us', 'de']:
eq_(schema['columnNames'], ['field1', 'field2'])
eq_(schema['columnTypes'], ['DOUBLE_TYPE', 'LONG_TYPE'])
else:
assert False
return jstr("ok")
return None
def mock_TwitterK(url, request):
if url.path == '/dataset':
fields = dict(request.body.fields)
relation_key = get_field(fields, 'relationKey')
schema = get_field(fields, 'schema')
overwrite = get_field(fields, 'overwrite')
if not overwrite:
return {
'status_code': 409,
'content': 'That dataset already exists.'
}
eq_(relation_key['userName'], 'test')
eq_(relation_key['programName'], 'testp')
if relation_key['relationName'] == 'TwitterK':
eq_(schema['columnNames'], ['src', 'dst'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'LONG_TYPE'])
elif relation_key['relationName'] == 'TwitterKnoheader':
eq_(schema['columnNames'], ['column0', 'column1'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'LONG_TYPE'])
elif relation_key['relationName'] == 'plaintext':
eq_(schema['columnNames'], ['number', 'value'])
eq_(schema['columnTypes'], ['LONG_TYPE', 'STRING_TYPE'])
elif relation_key['relationName'] == 'float':
eq_(schema['columnNames'], ['field1', 'field2'])
eq_(schema['columnTypes'], ['DOUBLE_TYPE', 'DOUBLE_TYPE'])
elif relation_key['relationName'] == 'nulls':
eq_(schema['columnNames'], ['field1', 'field2', 'field3'])
eq_(schema['columnTypes'],
['LONG_TYPE', 'STRING_TYPE', 'STRING_TYPE'])
elif relation_key['relationName'] in ['us', 'de']:
eq_(schema['columnNames'], ['field1', 'field2'])
eq_(schema['columnTypes'], ['DOUBLE_TYPE', 'LONG_TYPE'])
else:
assert False
return jstr("ok")
return None
def mock_myria_get(url, request):
query_params = urlparse.parse_qs(url.query)
# The below JSON responses are taken directly from production Myria instance
# with vega.cs.washington.edu:1776 changed to fakefake:12345
queries = [{
"url": "http://fake.fake:12345/query/query-140",
"queryId": 140,
"rawQuery": "JustX(x) :- TwitterK(x,y)",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-26T15:10:37.718-08:00",
"startTime": "2014-02-26T15:10:37.878-08:00",
"finishTime": "2014-02-26T15:10:38.648-08:00",
"message": None,
"elapsedNanos": 769418780,
"status": "SUCCESS"
}, {
"url": "http://fake.fake:12345/query/query-139",
"queryId": 139,
"rawQuery": "download [public#adhoc#TwitterK]",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-26T00:51:54.868-08:00",
"startTime": "2014-02-26T00:51:54.963-08:00",
"finishTime": "2014-02-26T00:51:55.002-08:00",
"message": None,
"elapsedNanos": 38598641,
"status": "SUCCESS"
}, {
"url": "http://fake.fake:12345/query/query-138",
"queryId": 138,
"rawQuery": "download [public#adhoc#Twitter]",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-26T00:36:54.845-08:00",
"startTime": "2014-02-26T00:36:54.970-08:00",
"finishTime": "2014-02-26T00:36:55.222-08:00",
"message": None,
"elapsedNanos": 252420874,
"status": "KILLED"
}, {
"url": "http://fake.fake:12345/query/query-137",
"queryId": 137,
"rawQuery": "download [public#adhoc#TwitterK]",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-26T00:10:06.066-08:00",
"startTime": "2014-02-26T00:10:06.173-08:00",
"finishTime": "2014-02-26T00:10:06.926-08:00",
"message": None,
"elapsedNanos": 752888288,
"status": "SUCCESS"
}, {
"url": "http://fake.fake:12345/query/query-136",
"queryId": 136,
"rawQuery": "download [public#adhoc#TwitterK]",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T23:59:33.023-08:00",
"startTime": "2014-02-25T23:59:33.181-08:00",
"finishTime": "2014-02-25T23:59:33.337-08:00",
"message": None,
"elapsedNanos": 155921007,
"status": "KILLED"
}, {
"url": "http://fake.fake:12345/query/query-135",
"queryId": 135,
"rawQuery": "download [public#__TEMP__#JustX]",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T23:55:20.921-08:00",
"startTime": "2014-02-25T23:55:21.126-08:00",
"finishTime": "2014-02-25T23:55:21.396-08:00",
"message": None,
"elapsedNanos": 270059319,
"status": "KILLED"
}, {
"url": "http://fake.fake:12345/query/query-134",
"queryId": 134,
"rawQuery": "JustX(x) :- TwitterK(x,y)",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T23:07:21.028-08:00",
"startTime": "2014-02-25T23:07:21.469-08:00",
"finishTime": "2014-02-25T23:07:22.765-08:00",
"message": None,
"elapsedNanos": 1295696742,
"status": "SUCCESS"
}, {
"url": "http://fake.fake:12345/query/query-133",
"queryId": 133,
"rawQuery": "JustX(x) :- TwitterK(x,y)",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T18:01:57.636-08:00",
"startTime": None,
"finishTime": None,
"message": None,
"elapsedNanos": None,
"status": "UNKNOWN"
}, {
"url": "http://fake.fake:12345/query/query-132",
"queryId": 132,
"rawQuery":
"all_opp_vct =\nSELECT Opp.*, Vct.pop, Vct.support\nFROM SCAN(armbrustlab:seaflow:all_opp_v3) AS Opp,\n SCAN(armbrustlab:seaflow:all_vct) AS Vct\nWHERE (Opp.Cruise = Vct.Cruise)\n AND (Opp.Day = Vct.Day)\n AND (Opp.File_Id = Vct.File_Id)\n AND (Opp.Cell_Id = Vct.Cell_Id);",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T17:05:18.617-08:00",
"startTime": None,
"finishTime": None,
"message": None,
"elapsedNanos": None,
"status": "UNKNOWN"
}, {
"url": "http://fake.fake:12345/query/query-131",
"queryId": 131,
"rawQuery":
"T1 = SCAN(TwitterK);\n\nT2 = [FROM T1 EMIT $0 AS x];\n\nSTORE (T2, JustX);",
"logicalRa": None,
"physicalPlan": None,
"submitTime": "2014-02-25T15:28:47.782-08:00",
"startTime": "2014-02-25T15:28:48.668-08:00",
"finishTime": "2014-02-25T15:28:50.177-08:00",
"message": None,
"elapsedNanos": 1509299867,
"status": "SUCCESS"
}]
twitter_dataset = {
"relationKey": {
"userName": "******",
"programName": "adhoc",
"relationName": "Twitter"
},
"schema": {
"columnTypes": ["INT_TYPE", "INT_TYPE"],
"columnNames": ["followee", "follower"]
},
"numTuples":
1427579976,
"queryId":
2,
"created":
"2014-02-09T12:40:43.438-08:00",
"howDistributed": {
"df": None,
"workers": None
},
"uri":
"http://fake.fake:12345/dataset/user-public/program-adhoc/relation-Twitter"
}
twitterk_dataset = {
"relationKey": {
"userName": "******",
"programName": "adhoc",
"relationName": "TwitterK"
},
"schema": {
"columnTypes": ["INT_TYPE", "INT_TYPE"],
"columnNames": ["followee", "follower"]
},
"numTuples":
2715,
"queryId":
3,
"created":
"2014-02-09T12:40:43.438-08:00",
"howDistributed": {
"df": None,
"workers": None
},
"uri":
"http://fake.fake:12345/dataset/user-public/program-adhoc/relation-TwitterK"
}
if url.path == '/workers':
return jstr({'1': 'localhost:12347', '2': 'localhost:12348'})
elif url.path == '/workers/alive':
return jstr([1, 2])
elif url.path == '/dataset':
return jstr([twitter_dataset, twitterk_dataset])
elif url.path == '/dataset/user-public/program-adhoc/relation-Twitter':
return jstr(twitter_dataset)
elif url.path == '/dataset/user-public/program-adhoc/relation-TwitterK':
return jstr(twitterk_dataset)
elif url.path == '/query':
limit = int((query_params.get('limit') or [10])[0])
ret = queries[:limit]
body = {
'max': ret[0]['queryId'],
'min': ret[-1]['queryId'],
'results': ret
}
return {'status_code': 200, 'content': body}
elif url.path == '/query/query-140':
return {
'status_code': 201,
'headers': {
'Location': 'http://fake.fake:12345/query/query-140'
},
'content': queries[0]
}
print >> sys.stderr, "Did not handle URL {}".format(url)