示例#1
0
    def setUp(self):
        # if GENIE_SETUP_JOBS=false, don't submit a job
        env = os.getenv('GENIE_SETUP_JOBS')
        setUpJobs = True
        if env is not None and env.lower() == 'false':
            setUpJobs = False
        if not setUpJobs:
            return

        # submit a job so the db is not empty, if needed
        payload = '''
        {
            "jobInfo":
            {
                "jobType": "hive",
                "jobName": "HIVE_TEST", 
                "schedule": "adHoc",
                "configuration": "test",
                "clusterName": "MY_CLUSTER_NAME",
                "clusterId": "MY_CLUSTER_ID",
                "cmdArgs": "-e THIS IS GOING TO FAIL",
                "userName": "******"
            }
        }
        '''
        try:
            restclient.post(serviceUrl=serviceUrl, payload=payload)
        except urllib2.HTTPError, e:
            pass
示例#2
0
 def setUp(self):
     # if GENIE_SETUP_JOBS=false, don't submit a job
     env = os.getenv('GENIE_SETUP_JOBS')
     setUpJobs = True
     if env is not None and env.lower() == 'false':
         setUpJobs = False 
     if not setUpJobs:
         return
     
     # submit a job so the db is not empty, if needed
     payload = '''
     {
         "jobInfo":
         {
             "jobType": "hive",
             "jobName": "HIVE_TEST", 
             "schedule": "adHoc",
             "configuration": "test",
             "clusterName": "MY_CLUSTER_NAME",
             "clusterId": "MY_CLUSTER_ID",
             "cmdArgs": "-e THIS IS GOING TO FAIL",
             "userName": "******"
         }
     }
     '''
     try:
         restclient.post(serviceUrl=serviceUrl, payload=payload)
     except urllib2.HTTPError, e:
         pass
示例#3
0
 def setUp(self):
     # create a new one, if needed
     payload = '''
     <request>
       <pigConfig>
         <id>''' + id + '''</id>
         <name>''' + name + '''</name>
         <type>''' + type + '''</type>
         <status>INACTIVE</status>
         <s3PigProperties>s3://BUCKET/PATH/TO/PIG.PROPERTIES</s3PigProperties>
         <user>testuser</user>
       </pigConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl,
                                    payload=payload,
                                    contentType='application/xml')
         obj = json.loads(response.read())
         assert obj['pigConfigs']['pigConfig'][
             'createTime'] > 0, "Create time not set correctly for config"
         assert obj['pigConfigs']['pigConfig'][
             'updateTime'] > 0, "Update time not set correctly for config"
     except urllib2.HTTPError, e:
         if e.code == 409:
             pass
         else:
             raise
示例#4
0
def submitJob(serviceUrl, payload, contentType='application/json'):
    response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType=contentType)
    results = response.read()
    json_obj = json.loads(results)
    jobID = json_obj[u'jobs'][u'jobInfo'][u'jobID']
    print "Job ID: ", jobID
    print "Output URI: ", json_obj[u'jobs'][u'jobInfo'][u'outputURI']
    print "Job submission successful"
    return jobID
示例#5
0
class ClusterConfigTestCase(unittest.TestCase):
    def setUp(self):
        # create a new hive config, if needed
        # create a new one, if needed
        payload = '''
        <request>
          <hiveConfig>
            <id>''' + hive_id + '''</id>
            <name>''' + hive_name + '''</name>
            <type>''' + hive_type + '''</type>
            <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
            <user>testuser</user>
          </hiveConfig>
        </request>
        '''
        try:
            response = restclient.post(serviceUrl=hiveServiceUrl,
                                       payload=payload,
                                       contentType='application/xml')
        except urllib2.HTTPError, e:
            if e.code == 409:
                pass
            else:
                raise

        # create a new cluster config, if needed
        payload = '''
        <request>
          <clusterConfig>
            <id>''' + id + '''</id>
            <name>''' + name + '''</name>
            <adHoc>false</adHoc>
            <unitTest>true</unitTest>
            <status>UP</status>
            <s3MapredSiteXml>s3://PATH/TO/MAPRED-SITE.XML</s3MapredSiteXml>
            <s3CoreSiteXml>s3://PATH/TO/CORE-SITE.XML</s3CoreSiteXml>
            <s3HdfsSiteXml>s3://PATH/TO/HDFS-SITE.XML</s3HdfsSiteXml>
            <unitTestHiveConfigId>unittest_hiveconfig_id</unitTestHiveConfigId>
            <user>testuser</user>
          </clusterConfig>
        </request>
        '''
        try:
            response = restclient.post(serviceUrl=serviceUrl,
                                       payload=payload,
                                       contentType='application/xml')
            obj = json.loads(response.read())
            assert obj['clusterConfigs']['clusterConfig'][
                'createTime'] > 0, "Create time not set correctly for config"
            assert obj['clusterConfigs']['clusterConfig'][
                'updateTime'] > 0, "Update time not set correctly for config"
        except urllib2.HTTPError, e:
            if e.code == 409:
                pass
            else:
                raise
示例#6
0
def postcase(case,sso_cookie):
    body = get_body(case)
    headers = case.headers
    if case.type == 'rest':
        resp = restclient.post(case.url, sso_cookie, body=body, args=case.args, parts=case.bodyParts, headers=headers)
    elif case.type == 'rpc':
        resp = rpc_post(case, sso_cookie, body)
    else:
        raise Exception('unkown case type %s' % case)
    return resp
示例#7
0
 def testMissingJobInfo(self):
     print "HTTP POST: missing job info: "
     payload = '''
     {
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#8
0
 def testMissingJobInfo(self):     
     print "HTTP POST: missing job info: "
     payload = '''
     {
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#9
0
 def testPostMissingConfig(self):
     print "HTTP POST: missing config element"        
     payload = '''
     <request>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#10
0
 def testPostMissingElement(self):
     print "HTTP GET: missing hiveConfig element"
     payload = '''
     <request>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#11
0
 def testPostMissingUser(self):     
     print "HTTP GET: missing user"
     payload = '''
     <request>
       <hiveConfig>
         <name>foo</name>
         <type>TEST</type>
         <s3HiveSiteXml>s3://bar</s3HiveSiteXml>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#12
0
 def testPostMissingUser(self):    
     print "HTTP POST: missing user"
     payload = '''
     <request>
       <clusterConfig>
         <name>prod</name>
         <test>true</test>
         <s3MapredSiteXml>s3://path/to/mapred-file</s3MapredSiteXml>
       </clusterConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#13
0
 def testPostMissingUser(self):     
     print "HTTP GET: missing user"
     payload = '''
     <request>
       <hiveConfig>
         <name>foo</name>
         <type>TEST</type>
         <s3HiveSiteXml>s3://bar</s3HiveSiteXml>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#14
0
 def testMissingCmdArgs(self):
     print "HTTP POST: missing command args: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#15
0
 def testMissingJobType(self):     
     print "HTTP POST: missing job type: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "schedule": "adHoc",
             "cmdArgs":"jar hadoop-examples-0.20.205.1.jar sleep -m 1 -mt 1" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#16
0
 def testMissingCmdArgs(self):    
     print "HTTP POST: missing command args: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#17
0
 def testPostBadType(self):    
     print "HTTP GET: bad type"
     payload = '''
     <request>
       <hiveConfig>
         <name>''' + name + '''</name>
         <type>BAD_TYPE</type>
         <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
         <user>testuser</user>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#18
0
 def testPostBadType(self):    
     print "HTTP GET: bad type"
     payload = '''
     <request>
       <hiveConfig>
         <name>''' + name + '''</name>
         <type>BAD_TYPE</type>
         <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
         <user>testuser</user>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#19
0
 def testMissingJobType(self):
     print "HTTP POST: missing job type: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "schedule": "adHoc",
             "cmdArgs":"jar hadoop-examples-0.20.205.1.jar sleep -m 1 -mt 1" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#20
0
 def testPostMissingUser(self):
     print "HTTP POST: missing user"
     payload = '''
     <request>
       <clusterConfig>
         <name>prod</name>
         <test>true</test>
         <s3MapredSiteXml>s3://path/to/mapred-file</s3MapredSiteXml>
       </clusterConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl,
                                    payload=payload,
                                    contentType='application/xml')
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#21
0
 def testIncorrectCmdArgs(self):
     print "HTTP POST: incorrect command-line args: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc",
             "configuration": "prod",
             "cmdArgs":"foo bar" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#22
0
def rpc_post(case,sso_cookie,body ):
    rpcbody = {}
    if case.zkHost is not None:
         registry = ZookeeperRegistry(case.zkHost)
         user_provider = DubboClient(case.reference, registry,group=case.group,version=case.version)
         hosts = registry.get_provides(case.reference, group=case.group,version=case.version)
         if len(hosts) < 1:
             raise Exception('zookeeper查询不到该服务的注册信息' % case)
         host = list(hosts.keys())[0]
         rpcbody['reference'] = 'dubbo://{0}/{1}'.format(host,case.reference)
    elif case.host is not None:
        rpcbody['reference'] = 'dubbo://{0}/{1}'.format(case.host,case.reference)
    else:
        raise Exception('rpc case zkhost 和 host 至少包含一个' % case)

    rpcbody['data'] = body#.decode("utf-8")
    rpcbody['version']=case.version
    rpcbody['group'] = case.group
    #rpcbody['reference'] = case.reference
    rpcbody['action'] = case.action
    return restclient.post(case.url, sso_cookie, body=str(rpcbody).encode("utf-8"), args=case.args, parts=case.bodyParts, headers=case.headers)
示例#23
0
 def setUp(self):
     # create a new hive config, if needed
     # create a new one, if needed
     payload = '''
     <request>
       <hiveConfig>
         <id>''' + hive_id + '''</id>
         <name>''' + hive_name + '''</name>
         <type>''' + hive_type + '''</type>
         <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
         <user>testuser</user>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=hiveServiceUrl, payload=payload, contentType='application/xml')
     except urllib2.HTTPError, e:
         if e.code == 409:
             pass
         else:
             raise
示例#24
0
 def testIncorrectCmdArgs(self):    
     print "HTTP POST: incorrect command-line args: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc",
             "configuration": "prod",
             "cmdArgs":"foo bar" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 400, "received unexpected unsuccessful response"
示例#25
0
 def testBadHiveConfigId(self):
     print "HTTP POST: bad hive config"
     payload = '''
     {
         "jobInfo":
         {
             "userName" : "genietest", 
             "jobType": "hive", 
             "configuration": "prod", 
             "schedule": "adHoc",
             "cmdArgs": "-f hive.q",
             "hiveConfigId": "DOES_NOT_EXIST"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
示例#26
0
 def testMissingClusterId(self):
     print "HTTP POST: cluster id doesn't exist: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc",
             "configuration": "prod",
             "cmdArgs": "-f hive.q",
             "clusterId": "DOES_NOT_EXIST" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 402, "received unexpected unsuccessful response"
示例#27
0
 def testBadPigVersion(self):
     print "HTTP POST: bad pig version"
     payload = '''
     {
         "jobInfo":
         {
             "userName" : "genietest", 
             "jobType": "pig", 
             "configuration": "prod", 
             "schedule": "adHoc",
             "cmdArgs":"-f pig.q",
             "pigVersion": "0.0"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
示例#28
0
 def testBadPigVersion(self):
     print "HTTP POST: bad pig version"
     payload = '''
     {
         "jobInfo":
         {
             "userName" : "genietest", 
             "jobType": "pig", 
             "configuration": "prod", 
             "schedule": "adHoc",
             "cmdArgs":"-f pig.q",
             "pigVersion": "0.0"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
示例#29
0
 def testBadHiveConfigId(self):
     print "HTTP POST: bad hive config"
     payload = '''
     {
         "jobInfo":
         {
             "userName" : "genietest", 
             "jobType": "hive", 
             "configuration": "prod", 
             "schedule": "adHoc",
             "cmdArgs": "-f hive.q",
             "hiveConfigId": "DOES_NOT_EXIST"
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 500 or e.code == 402, "received unexpected unsuccessful response"
示例#30
0
 def testMissingClusterId(self):
     print "HTTP POST: cluster id doesn't exist: "
     payload = '''
     {
         "jobInfo":
         {
             "userName": "******",
             "jobType": "hive",
             "schedule": "adhoc",
             "configuration": "prod",
             "cmdArgs": "-f hive.q",
             "clusterId": "DOES_NOT_EXIST" 
         }
     }
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload)
         print response.read()
         assert False, "received unexpected successful response"
     except urllib2.HTTPError, e:
         print e.read()
         assert e.code == 402, "received unexpected unsuccessful response"
示例#31
0
 def setUp(self):
     # create a new hive config, if needed
     # create a new one, if needed
     payload = '''
     <request>
       <hiveConfig>
         <id>''' + hive_id + '''</id>
         <name>''' + hive_name + '''</name>
         <type>''' + hive_type + '''</type>
         <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
         <user>testuser</user>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=hiveServiceUrl,
                                    payload=payload,
                                    contentType='application/xml')
     except urllib2.HTTPError, e:
         if e.code == 409:
             pass
         else:
             raise
示例#32
0
 def setUp(self):
     # create a new one, if needed
     payload = '''
     <request>
       <hiveConfig>
         <id>''' + id + '''</id>
         <name>''' + name + '''</name>
         <type>''' + type + '''</type>
         <status>INACTIVE</status>
         <s3HiveSiteXml>s3://BUCKET/PATH/TO/HIVE-SITE.XML</s3HiveSiteXml>
         <user>testuser</user>
       </hiveConfig>
     </request>
     '''
     try:
         response = restclient.post(serviceUrl=serviceUrl, payload=payload, contentType='application/xml')
         obj = json.loads(response.read())
         assert obj['hiveConfigs']['hiveConfig']['createTime'] > 0, "Create time not set correctly for config"
         assert obj['hiveConfigs']['hiveConfig']['updateTime'] > 0, "Update time not set correctly for config"
     except urllib2.HTTPError, e:
         if e.code == 409:
             pass
         else:
             raise