def setUp(self):
# if GENIE_SETUP_JOBS=false, don't submit a job
env = os.getenv('GENIE_SETUP_JOBS')
setUpJobs = True
if env is not None and env.lower() == 'false':
setUpJobs = False
if not setUpJobs:
return
# submit a job so the db is not empty, if needed
payload = '''
{
"jobInfo":
{
"jobType": "hive",
"jobName": "HIVE_TEST",
"schedule": "adHoc",
"configuration": "test",
"clusterName": "MY_CLUSTER_NAME",
"clusterId": "MY_CLUSTER_ID",
"cmdArgs": "-e THIS IS GOING TO FAIL",
"userName": "******"
}
}
'''
try:
restclient.post(serviceUrl=serviceUrl, payload=payload)
except urllib2.HTTPError, e:
pass
def setUp(self):
# if GENIE_SETUP_JOBS=false, don't submit a job
env = os.getenv('GENIE_SETUP_JOBS')
setUpJobs = True
if env is not None and env.lower() == 'false':
setUpJobs = False
if not setUpJobs:
return
# submit a job so the db is not empty, if needed
payload = '''
{
"jobInfo":
{
"jobType": "hive",
"jobName": "HIVE_TEST",
"schedule": "adHoc",
"configuration": "test",
"clusterName": "MY_CLUSTER_NAME",
"clusterId": "MY_CLUSTER_ID",
"cmdArgs": "-e THIS IS GOING TO FAIL",
"userName": "******"
}
}
'''
try:
restclient.post(serviceUrl=serviceUrl, payload=payload)
except urllib2.HTTPError, e:
pass
def setUp(self):
# create a new one, if needed
payload = '''
<request>
<pigConfig>
<id>''' + id + '''</id>
<name>''' + name + '''</name>
<type>''' + type + '''</type>
<status>INACTIVE</status>
<s3PigProperties>s3://BUCKET/PATH/TO/PIG.PROPERTIES</s3PigProperties>
<user>testuser</user>
</pigConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl,
payload=payload,
contentType='application/xml')
obj = json.loads(response.read())
assert obj['pigConfigs']['pigConfig'][
'createTime'] > 0, "Create time not set correctly for config"
assert obj['pigConfigs']['pigConfig'][
'updateTime'] > 0, "Update time not set correctly for config"
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise
def submitJob(serviceUrl, payload, contentType='application/json'):
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType=contentType)
results = response.read()
json_obj = json.loads(results)
jobID = json_obj[u'jobs'][u'jobInfo'][u'jobID']
print "Job ID: ", jobID
print "Output URI: ", json_obj[u'jobs'][u'jobInfo'][u'outputURI']
print "Job submission successful"
return jobID
class ClusterConfigTestCase(unittest.TestCase):
def setUp(self):
# create a new hive config, if needed
# create a new one, if needed
payload = '''
<request>
<hiveConfig>
<id>''' + hive_id + '''</id>
<name>''' + hive_name + '''</name>
<type>''' + hive_type + '''</type>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=hiveServiceUrl,
payload=payload,
contentType='application/xml')
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise
# create a new cluster config, if needed
payload = '''
<request>
<clusterConfig>
<id>''' + id + '''</id>
<name>''' + name + '''</name>
<adHoc>false</adHoc>
<unitTest>true</unitTest>
<status>UP</status>
<s3MapredSiteXml>s3://PATH/TO/MAPRED-SITE.XML</s3MapredSiteXml>
<s3CoreSiteXml>s3://PATH/TO/CORE-SITE.XML</s3CoreSiteXml>
<s3HdfsSiteXml>s3://PATH/TO/HDFS-SITE.XML</s3HdfsSiteXml>
<unitTestHiveConfigId>unittest_hiveconfig_id</unitTestHiveConfigId>
<user>testuser</user>
</clusterConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl,
payload=payload,
contentType='application/xml')
obj = json.loads(response.read())
assert obj['clusterConfigs']['clusterConfig'][
'createTime'] > 0, "Create time not set correctly for config"
assert obj['clusterConfigs']['clusterConfig'][
'updateTime'] > 0, "Update time not set correctly for config"
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise
def postcase(case,sso_cookie):
body = get_body(case)
headers = case.headers
if case.type == 'rest':
resp = restclient.post(case.url, sso_cookie, body=body, args=case.args, parts=case.bodyParts, headers=headers)
elif case.type == 'rpc':
resp = rpc_post(case, sso_cookie, body)
else:
raise Exception('unkown case type %s' % case)
return resp
def testMissingJobInfo(self):
print "HTTP POST: missing job info: "
payload = '''
{
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testMissingJobInfo(self):
print "HTTP POST: missing job info: "
payload = '''
{
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingConfig(self):
print "HTTP POST: missing config element"
payload = '''
<request>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingElement(self):
print "HTTP GET: missing hiveConfig element"
payload = '''
<request>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingUser(self):
print "HTTP GET: missing user"
payload = '''
<request>
<hiveConfig>
<name>foo</name>
<type>TEST</type>
<s3HiveSiteXml>s3://bar</s3HiveSiteXml>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingUser(self):
print "HTTP POST: missing user"
payload = '''
<request>
<clusterConfig>
<name>prod</name>
<test>true</test>
<s3MapredSiteXml>s3://path/to/mapred-file</s3MapredSiteXml>
</clusterConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingUser(self):
print "HTTP GET: missing user"
payload = '''
<request>
<hiveConfig>
<name>foo</name>
<type>TEST</type>
<s3HiveSiteXml>s3://bar</s3HiveSiteXml>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testMissingCmdArgs(self):
print "HTTP POST: missing command args: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testMissingJobType(self):
print "HTTP POST: missing job type: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"schedule": "adHoc",
"cmdArgs":"jar hadoop-examples-0.20.205.1.jar sleep -m 1 -mt 1"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testMissingCmdArgs(self):
print "HTTP POST: missing command args: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostBadType(self):
print "HTTP GET: bad type"
payload = '''
<request>
<hiveConfig>
<name>''' + name + '''</name>
<type>BAD_TYPE</type>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostBadType(self):
print "HTTP GET: bad type"
payload = '''
<request>
<hiveConfig>
<name>''' + name + '''</name>
<type>BAD_TYPE</type>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testMissingJobType(self):
print "HTTP POST: missing job type: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"schedule": "adHoc",
"cmdArgs":"jar hadoop-examples-0.20.205.1.jar sleep -m 1 -mt 1"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testPostMissingUser(self):
print "HTTP POST: missing user"
payload = '''
<request>
<clusterConfig>
<name>prod</name>
<test>true</test>
<s3MapredSiteXml>s3://path/to/mapred-file</s3MapredSiteXml>
</clusterConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl,
payload=payload,
contentType='application/xml')
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testIncorrectCmdArgs(self):
print "HTTP POST: incorrect command-line args: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc",
"configuration": "prod",
"cmdArgs":"foo bar"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def rpc_post(case,sso_cookie,body ):
rpcbody = {}
if case.zkHost is not None:
registry = ZookeeperRegistry(case.zkHost)
user_provider = DubboClient(case.reference, registry,group=case.group,version=case.version)
hosts = registry.get_provides(case.reference, group=case.group,version=case.version)
if len(hosts) < 1:
raise Exception('zookeeper查询不到该服务的注册信息' % case)
host = list(hosts.keys())[0]
rpcbody['reference'] = 'dubbo://{0}/{1}'.format(host,case.reference)
elif case.host is not None:
rpcbody['reference'] = 'dubbo://{0}/{1}'.format(case.host,case.reference)
else:
raise Exception('rpc case zkhost 和 host 至少包含一个' % case)
rpcbody['data'] = body#.decode("utf-8")
rpcbody['version']=case.version
rpcbody['group'] = case.group
#rpcbody['reference'] = case.reference
rpcbody['action'] = case.action
return restclient.post(case.url, sso_cookie, body=str(rpcbody).encode("utf-8"), args=case.args, parts=case.bodyParts, headers=case.headers)
def setUp(self):
# create a new hive config, if needed
# create a new one, if needed
payload = '''
<request>
<hiveConfig>
<id>''' + hive_id + '''</id>
<name>''' + hive_name + '''</name>
<type>''' + hive_type + '''</type>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=hiveServiceUrl, payload=payload, contentType='application/xml')
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise
def testIncorrectCmdArgs(self):
print "HTTP POST: incorrect command-line args: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc",
"configuration": "prod",
"cmdArgs":"foo bar"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 400, "received unexpected unsuccessful response"
def testBadHiveConfigId(self):
print "HTTP POST: bad hive config"
payload = '''
{
"jobInfo":
{
"userName" : "genietest",
"jobType": "hive",
"configuration": "prod",
"schedule": "adHoc",
"cmdArgs": "-f hive.q",
"hiveConfigId": "DOES_NOT_EXIST"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
def testMissingClusterId(self):
print "HTTP POST: cluster id doesn't exist: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc",
"configuration": "prod",
"cmdArgs": "-f hive.q",
"clusterId": "DOES_NOT_EXIST"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 402, "received unexpected unsuccessful response"
def testBadPigVersion(self):
print "HTTP POST: bad pig version"
payload = '''
{
"jobInfo":
{
"userName" : "genietest",
"jobType": "pig",
"configuration": "prod",
"schedule": "adHoc",
"cmdArgs":"-f pig.q",
"pigVersion": "0.0"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
def testBadPigVersion(self):
print "HTTP POST: bad pig version"
payload = '''
{
"jobInfo":
{
"userName" : "genietest",
"jobType": "pig",
"configuration": "prod",
"schedule": "adHoc",
"cmdArgs":"-f pig.q",
"pigVersion": "0.0"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
def testBadHiveConfigId(self):
print "HTTP POST: bad hive config"
payload = '''
{
"jobInfo":
{
"userName" : "genietest",
"jobType": "hive",
"configuration": "prod",
"schedule": "adHoc",
"cmdArgs": "-f hive.q",
"hiveConfigId": "DOES_NOT_EXIST"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
def testMissingClusterId(self):
print "HTTP POST: cluster id doesn't exist: "
payload = '''
{
"jobInfo":
{
"userName": "******",
"jobType": "hive",
"schedule": "adhoc",
"configuration": "prod",
"cmdArgs": "-f hive.q",
"clusterId": "DOES_NOT_EXIST"
}
}
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload)
print response.read()
assert False, "received unexpected successful response"
except urllib2.HTTPError, e:
print e.read()
assert e.code == 402, "received unexpected unsuccessful response"
def setUp(self):
# create a new hive config, if needed
# create a new one, if needed
payload = '''
<request>
<hiveConfig>
<id>''' + hive_id + '''</id>
<name>''' + hive_name + '''</name>
<type>''' + hive_type + '''</type>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=hiveServiceUrl,
payload=payload,
contentType='application/xml')
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise
def setUp(self):
# create a new one, if needed
payload = '''
<request>
<hiveConfig>
<id>''' + id + '''</id>
<name>''' + name + '''</name>
<type>''' + type + '''</type>
<status>INACTIVE</status>
<s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
<user>testuser</user>
</hiveConfig>
</request>
'''
try:
response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
obj = json.loads(response.read())
assert obj['hiveConfigs']['hiveConfig']['createTime'] > 0, "Create time not set correctly for config"
assert obj['hiveConfigs']['hiveConfig']['updateTime'] > 0, "Update time not set correctly for config"
except urllib2.HTTPError, e:
if e.code == 409:
pass
else:
raise