def main():
    upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)

    # Submit job to batch compute.
    clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
    job_desc = get_job_desc(cfg.FULL_PACKAGE)
    job = clnt.create_job(job_desc)

    t = 10
    print('Sleep %s second, please wait.'%t)
    time.sleep(t)

    # Wait for jobs terminated.
    while(True):
        s = clnt.get_job(job)
        if s.State in ['Waiting', 'Running']:
            print('Job %s is now %s'%(job, s.State))
            time.sleep(3)
            continue
        else:
            # 'Failed', 'Stopped', 'Finished'
            print('Job %s is now %s'%(job, s.State))
            if s.State == 'Finished':
                result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH)
                # Print out all prime numbers from 0 to 10000.
                print result.splitlines()
            break
    clnt.delete_job(job)
Esempio n. 2
0
def main():
    upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)

    # Submit job to batch compute.
    clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
    job_json = get_job_desc(cfg.FULL_PACKAGE)
    job = clnt.create_job(job_json)

    t = 10
    print('Sleep %s second, please wait.'%t)
    time.sleep(t)

    # Wait for jobs terminated.
    while(True):
        s = clnt.get_job(job)
        if s.State in ['Waiting', 'Running']:
            print('Job %s is now %s'%(job, s.State))
            time.sleep(3)
            continue
        else:
            # 'Failed', 'Stopped', 'Finished'
            print('Job %s is now %s'%(job, s.State))
            if s.State == 'Finished':
                oss_clnt = SimpleOss(cfg.OSS_HOST, cfg.ID, cfg.KEY)
                result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH)
                # Print out all prime numbers from 0 to 10000.
                print result.splitlines()
            break
    clnt.delete_job(job)
def main():
    upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)

    # Submit job to batch compute.
    clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
    job_desc = get_job_desc(cfg.FULL_PACKAGE)
    job = clnt.create_job(job_desc)
    t = 10
    print 'Sleep %s second, please wait.' % t
    time.sleep(t)

    # Wait for job Terminated.
    while (True):
        s = clnt.get_job(job)
        if s.State in ['Waiting', 'Running']:
            print('Job %s is now %s' % (job, s.State))
            time.sleep(3)
            continue
        else:
            # 'Failed', 'Stopped', 'Finished'
            print('Job %s is now %s' % (job, s.State))
            if s.State == 'Finished':
                result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH)
                # Print out the count of prime numbers from 0 to 10000.
                print('Total %s prime numbers from %s to %s.' %
                      (result, cfg.DATA_START, cfg.DATA_END))
            break
    # Release job from batchcompute.
    clnt.delete_job(job)
def main():
    upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)

    # Submit job to batch compute.
    clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
    job_json = get_job_desc(cfg.FULL_PACKAGE)
    job = clnt.create_job(job_json)

    t = 5
    print 'Sleep %s second, please wait.' % t
    time.sleep(t)

    # Wait for job terminated.
    while (True):
        s = clnt.get_job(job)
        if s.State in ['Waiting']:
            print('Job %s is now %s' % (job, s.State))
            time.sleep(10)
            t += 1
            continue
        elif s.State in ['Running']:
            print "Waiting%s s" % t
            t = 0
            print('Job %s is now %s' % (job, s.State))
            time.sleep(10)
            t += 1
        else:
            # 'Failed', 'Stopped', 'Finished'
            print('Job %s is now %s' % (job, s.State))
            print "running%s s" % t
            if s.State == 'Finished':
                # Print out total prime numbers from 0 to 10000.
                result = oss_clnt.download_str(cfg.OSS_BUCKET,
                                               cfg.COUNT_OUTPUT_PATH)
                print('Total %s prime numbers from %s to %s.' %
                      (result, 1, 10000))
            break
    clnt.delete_job(job)
def main():
    upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)

    # Submit job to batch compute.
    clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
    job_json = get_job_desc(cfg.FULL_PACKAGE)
    job = clnt.create_job(job_json)

    t = 5
    print 'Sleep %s second, please wait.'%t
    time.sleep(t)

    # Wait for job terminated.
    while(True):
        s = clnt.get_job(job)
        if s.State in ['Waiting']:
            print('Job %s is now %s'%(job, s.State))
            time.sleep(10)
            t+=1
            continue
        elif s.State in['Running']:
            print "Waiting%s s"%t
            t=0
            print('Job %s is now %s'%(job, s.State))
            time.sleep(10)
            t+=1
        else:
            # 'Failed', 'Stopped', 'Finished'
            print('Job %s is now %s'%(job, s.State))
            print "running%s s"%t
            if s.State == 'Finished':
                # Print out total prime numbers from 0 to 10000.
                result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.COUNT_OUTPUT_PATH)
                print('Total %s prime numbers from %s to %s.'%(result, 1, 10000))
            break
    clnt.delete_job(job)
Esempio n. 6
0
class ClientE2ETest(unittest.TestCase):
    def setUp(self):
        self.endpoint = CN_QINGDAO
        self.access_key_id = ""
        self.access_key_secret = ""
        self.image_id = ''

        self.client = Client(self.endpoint,
                             self.access_key_id,
                             self.access_key_secret,
                             human_readable=True)
        self.job_id = None

    def _get_job_desc(self):
        job_desc = JobDescription()
        find_task = TaskDescription()
        res_desc = ResourceDescription()

        find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz"
        find_task.ProgramName = 'find_prime_worker.py'
        find_task.ProgramType = 'python'
        find_task.ImageId = self.image_id
        find_task.InstanceCount = 3
        find_task.EnvironmentVariables = {}
        find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
        find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
        find_task.ResourceDescription = res_desc

        # Create count task.
        count_task = TaskDescription(find_task)
        count_task['InstanceCount'] = 1

        # Create task dag.
        task_dag = TaskDag()
        task_dag.add_task(task_name='Find', task=find_task)
        task_dag.add_task(task_name='Count', task=count_task)
        task_dag.Dependencies = {'Find': ['Count']}

        # count prime job description.
        job_desc.TaskDag = task_dag
        job_desc.JobName = 'PythonSDK'
        job_desc.Priority = 0
        return job_desc

    def tearDown(self):
        while True:
            if self.job_id and self.job_id in self.client.list_jobs():
                state = self.client.get_job(self.job_id).State
                if state in ['Waiting', 'Running']:
                    self.client.stop_job(self.job_id)
                else:
                    self.client.delete_job(self.job_id)
            else:
                break
        self.job_id = None

    def test_create_job(self):
        job_desc = self._get_job_desc()

        # Create jobs.
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        self.assertTrue(self.job_id)

    def test_stop_job(self):
        job_desc = self._get_job_desc()

        job = self.client.create_job(job_desc)
        self.job_id = job.JobId
        state = self.client.get_job(job).State
        if state in ['Waiting', 'Running']:
            self.client.stop_job(job)
        self.assertRaises(ClientError, self.client.stop_job, job)

    def test_update_priority(self):
        old_priority = 100
        new_priority = 200

        job_desc = self._get_job_desc()
        job_desc.Priority = old_priority

        # Create jobs.
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId
        status = self.client.get_job(job)
        self.assertEqual(status.Priority, old_priority)

        # update priority.
        self.assertRaises(ClientError, self.client.update_job_priority, job,
                          new_priority)
        try:
            self.client.update_job_priority(job, new_priority)
        except ClientError as e:
            code = e.get_code()
            msg = e.get_msg()
            request_id = e.get_requestid()
        else:
            self.assertFalse(False, 'ClientError should be raised')
        status = self.client.get_job(job)
        if status.State in ['Waiting', 'Running']:
            self.client.stop_job(job)
        self.client.update_job_priority(job, new_priority)
        status = self.client.get_job(job)
        self.assertEqual(status.Priority, new_priority)

    def test_start_job(self):
        job_desc = self._get_job_desc()
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        self.assertRaises(ClientError, self.client.start_job, job)
        status = self.client.get_job(job)
        if status.State in ['Waiting', 'Running']:
            self.client.stop_job(job)

    def test_list_images(self):
        image_list = self.client.list_images()
        for img in image_list:
            self.assertTrue(hasattr(img, 'ImageId'))
            self.assertTrue(not hasattr(img, 'ResourceId'))
            print(img.ImageId)

    def test_list_tasks(self):
        job_desc = self._get_job_desc()
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        task_list = self.client.list_tasks(self.job_id)
        for task in task_list:
            print(task.TaskName)
            for instance in task.InstanceList:
                print(instance.InstanceId)
Esempio n. 7
0
class ClientE2ETest(unittest.TestCase):
    def setUp(self):
        self.endpoint = CN_QINGDAO
        self.access_key_id = ""
        self.access_key_secret = ""
        self.image_id = ''

        self.client = Client(self.endpoint, self.access_key_id,
                        self.access_key_secret, human_readable=True)
        self.job_id = None

    def _get_job_desc(self):
        job_desc = JobDescription()
        find_task = TaskDescription()
        res_desc = ResourceDescription()

        find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz"
        find_task.ProgramName = 'find_prime_worker.py'
        find_task.ProgramType = 'python'
        find_task.ImageId = self.image_id
        find_task.InstanceCount = 3
        find_task.EnvironmentVariables = {}
        find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
        find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
        find_task.ResourceDescription = res_desc

        # Create count task. 
        count_task = TaskDescription(find_task)
        count_task['InstanceCount'] = 1

        # Create task dag.
        task_dag = TaskDag()
        task_dag.add_task(task_name='Find', task=find_task)
        task_dag.add_task(task_name='Count', task=count_task)
        task_dag.Dependencies = {
            'Find': ['Count']
        }

        # count prime job description.
        job_desc.TaskDag = task_dag
        job_desc.JobName = 'PythonSDK'
        job_desc.Priority = 0
        return job_desc

    def tearDown(self):
        while True:
            if self.job_id and self.job_id in self.client.list_jobs():
                state = self.client.get_job(self.job_id).State
                if state in ['Waiting', 'Running']:
                    self.client.stop_job(self.job_id)
                else:
                    self.client.delete_job(self.job_id)
            else:
                break
        self.job_id = None


    def test_create_job(self):
        job_desc = self._get_job_desc()

        # Create jobs.
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        self.assertTrue(self.job_id)

    def test_stop_job(self):
        job_desc = self._get_job_desc()

        job = self.client.create_job(job_desc)
        self.job_id = job.JobId
        state = self.client.get_job(job).State
        if state in ['Waiting', 'Running']:
            self.client.stop_job(job)
        self.assertRaises(ClientError, self.client.stop_job, job)

    def test_update_priority(self):
        old_priority = 100
        new_priority = 200

        job_desc = self._get_job_desc()
        job_desc.Priority = old_priority

        # Create jobs.
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId
        status = self.client.get_job(job)
        self.assertEqual(status.Priority, old_priority)

        # update priority.
        self.assertRaises(ClientError, self.client.update_job_priority, job, new_priority)
        try:
            self.client.update_job_priority(job, new_priority)
        except ClientError as e:
            code = e.get_code()
            msg = e.get_msg()
            request_id = e.get_requestid()
        else:
            self.assertFalse(False, 'ClientError should be raised')
        status = self.client.get_job(job)
        if status.State in ['Waiting', 'Running']:
            self.client.stop_job(job)
        self.client.update_job_priority(job, new_priority)
        status = self.client.get_job(job)
        self.assertEqual(status.Priority, new_priority)

    def test_start_job(self):
        job_desc = self._get_job_desc()
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        self.assertRaises(ClientError, self.client.start_job, job)
        status = self.client.get_job(job)
        if status.State in ['Waiting', 'Running']:
            self.client.stop_job(job)

    def test_list_images(self):
        image_list = self.client.list_images()
        for img in image_list:
            self.assertTrue(hasattr(img, 'ImageId'))
            self.assertTrue(not hasattr(img, 'ResourceId'))
            print(img.ImageId)

    def test_list_tasks(self):
        job_desc = self._get_job_desc()
        job = self.client.create_job(job_desc)
        self.job_id = job.JobId

        task_list = self.client.list_tasks(self.job_id)
        for task in task_list:
            print(task.TaskName)
            for instance in task.InstanceList:
                print(instance.InstanceId)