def main(): upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH) # Submit job to batch compute. clnt = Client(cfg.REGION, cfg.ID, cfg.KEY) job_desc = get_job_desc(cfg.FULL_PACKAGE) job = clnt.create_job(job_desc) t = 10 print('Sleep %s second, please wait.'%t) time.sleep(t) # Wait for jobs terminated. while(True): s = clnt.get_job(job) if s.State in ['Waiting', 'Running']: print('Job %s is now %s'%(job, s.State)) time.sleep(3) continue else: # 'Failed', 'Stopped', 'Finished' print('Job %s is now %s'%(job, s.State)) if s.State == 'Finished': result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH) # Print out all prime numbers from 0 to 10000. print result.splitlines() break clnt.delete_job(job)
def main(): upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH) # Submit job to batch compute. clnt = Client(cfg.REGION, cfg.ID, cfg.KEY) job_json = get_job_desc(cfg.FULL_PACKAGE) job = clnt.create_job(job_json) t = 10 print('Sleep %s second, please wait.'%t) time.sleep(t) # Wait for jobs terminated. while(True): s = clnt.get_job(job) if s.State in ['Waiting', 'Running']: print('Job %s is now %s'%(job, s.State)) time.sleep(3) continue else: # 'Failed', 'Stopped', 'Finished' print('Job %s is now %s'%(job, s.State)) if s.State == 'Finished': oss_clnt = SimpleOss(cfg.OSS_HOST, cfg.ID, cfg.KEY) result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH) # Print out all prime numbers from 0 to 10000. print result.splitlines() break clnt.delete_job(job)
def main(): upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH) # Submit job to batch compute. clnt = Client(cfg.REGION, cfg.ID, cfg.KEY) job_desc = get_job_desc(cfg.FULL_PACKAGE) job = clnt.create_job(job_desc) t = 10 print 'Sleep %s second, please wait.' % t time.sleep(t) # Wait for job Terminated. while (True): s = clnt.get_job(job) if s.State in ['Waiting', 'Running']: print('Job %s is now %s' % (job, s.State)) time.sleep(3) continue else: # 'Failed', 'Stopped', 'Finished' print('Job %s is now %s' % (job, s.State)) if s.State == 'Finished': result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH) # Print out the count of prime numbers from 0 to 10000. print('Total %s prime numbers from %s to %s.' % (result, cfg.DATA_START, cfg.DATA_END)) break # Release job from batchcompute. clnt.delete_job(job)
def main(): upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH) # Submit job to batch compute. clnt = Client(cfg.REGION, cfg.ID, cfg.KEY) job_json = get_job_desc(cfg.FULL_PACKAGE) job = clnt.create_job(job_json) t = 5 print 'Sleep %s second, please wait.' % t time.sleep(t) # Wait for job terminated. while (True): s = clnt.get_job(job) if s.State in ['Waiting']: print('Job %s is now %s' % (job, s.State)) time.sleep(10) t += 1 continue elif s.State in ['Running']: print "Waiting%s s" % t t = 0 print('Job %s is now %s' % (job, s.State)) time.sleep(10) t += 1 else: # 'Failed', 'Stopped', 'Finished' print('Job %s is now %s' % (job, s.State)) print "running%s s" % t if s.State == 'Finished': # Print out total prime numbers from 0 to 10000. result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.COUNT_OUTPUT_PATH) print('Total %s prime numbers from %s to %s.' % (result, 1, 10000)) break clnt.delete_job(job)
def main(): upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH) # Submit job to batch compute. clnt = Client(cfg.REGION, cfg.ID, cfg.KEY) job_json = get_job_desc(cfg.FULL_PACKAGE) job = clnt.create_job(job_json) t = 5 print 'Sleep %s second, please wait.'%t time.sleep(t) # Wait for job terminated. while(True): s = clnt.get_job(job) if s.State in ['Waiting']: print('Job %s is now %s'%(job, s.State)) time.sleep(10) t+=1 continue elif s.State in['Running']: print "Waiting%s s"%t t=0 print('Job %s is now %s'%(job, s.State)) time.sleep(10) t+=1 else: # 'Failed', 'Stopped', 'Finished' print('Job %s is now %s'%(job, s.State)) print "running%s s"%t if s.State == 'Finished': # Print out total prime numbers from 0 to 10000. result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.COUNT_OUTPUT_PATH) print('Total %s prime numbers from %s to %s.'%(result, 1, 10000)) break clnt.delete_job(job)
class ClientE2ETest(unittest.TestCase): def setUp(self): self.endpoint = CN_QINGDAO self.access_key_id = "" self.access_key_secret = "" self.image_id = '' self.client = Client(self.endpoint, self.access_key_id, self.access_key_secret, human_readable=True) self.job_id = None def _get_job_desc(self): job_desc = JobDescription() find_task = TaskDescription() res_desc = ResourceDescription() find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz" find_task.ProgramName = 'find_prime_worker.py' find_task.ProgramType = 'python' find_task.ImageId = self.image_id find_task.InstanceCount = 3 find_task.EnvironmentVariables = {} find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/" find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/" find_task.ResourceDescription = res_desc # Create count task. count_task = TaskDescription(find_task) count_task['InstanceCount'] = 1 # Create task dag. task_dag = TaskDag() task_dag.add_task(task_name='Find', task=find_task) task_dag.add_task(task_name='Count', task=count_task) task_dag.Dependencies = {'Find': ['Count']} # count prime job description. job_desc.TaskDag = task_dag job_desc.JobName = 'PythonSDK' job_desc.Priority = 0 return job_desc def tearDown(self): while True: if self.job_id and self.job_id in self.client.list_jobs(): state = self.client.get_job(self.job_id).State if state in ['Waiting', 'Running']: self.client.stop_job(self.job_id) else: self.client.delete_job(self.job_id) else: break self.job_id = None def test_create_job(self): job_desc = self._get_job_desc() # Create jobs. job = self.client.create_job(job_desc) self.job_id = job.JobId self.assertTrue(self.job_id) def test_stop_job(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId state = self.client.get_job(job).State if state in ['Waiting', 'Running']: self.client.stop_job(job) self.assertRaises(ClientError, self.client.stop_job, job) def test_update_priority(self): old_priority = 100 new_priority = 200 job_desc = self._get_job_desc() job_desc.Priority = old_priority # Create jobs. job = self.client.create_job(job_desc) self.job_id = job.JobId status = self.client.get_job(job) self.assertEqual(status.Priority, old_priority) # update priority. self.assertRaises(ClientError, self.client.update_job_priority, job, new_priority) try: self.client.update_job_priority(job, new_priority) except ClientError as e: code = e.get_code() msg = e.get_msg() request_id = e.get_requestid() else: self.assertFalse(False, 'ClientError should be raised') status = self.client.get_job(job) if status.State in ['Waiting', 'Running']: self.client.stop_job(job) self.client.update_job_priority(job, new_priority) status = self.client.get_job(job) self.assertEqual(status.Priority, new_priority) def test_start_job(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId self.assertRaises(ClientError, self.client.start_job, job) status = self.client.get_job(job) if status.State in ['Waiting', 'Running']: self.client.stop_job(job) def test_list_images(self): image_list = self.client.list_images() for img in image_list: self.assertTrue(hasattr(img, 'ImageId')) self.assertTrue(not hasattr(img, 'ResourceId')) print(img.ImageId) def test_list_tasks(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId task_list = self.client.list_tasks(self.job_id) for task in task_list: print(task.TaskName) for instance in task.InstanceList: print(instance.InstanceId)
class ClientE2ETest(unittest.TestCase): def setUp(self): self.endpoint = CN_QINGDAO self.access_key_id = "" self.access_key_secret = "" self.image_id = '' self.client = Client(self.endpoint, self.access_key_id, self.access_key_secret, human_readable=True) self.job_id = None def _get_job_desc(self): job_desc = JobDescription() find_task = TaskDescription() res_desc = ResourceDescription() find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz" find_task.ProgramName = 'find_prime_worker.py' find_task.ProgramType = 'python' find_task.ImageId = self.image_id find_task.InstanceCount = 3 find_task.EnvironmentVariables = {} find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/" find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/" find_task.ResourceDescription = res_desc # Create count task. count_task = TaskDescription(find_task) count_task['InstanceCount'] = 1 # Create task dag. task_dag = TaskDag() task_dag.add_task(task_name='Find', task=find_task) task_dag.add_task(task_name='Count', task=count_task) task_dag.Dependencies = { 'Find': ['Count'] } # count prime job description. job_desc.TaskDag = task_dag job_desc.JobName = 'PythonSDK' job_desc.Priority = 0 return job_desc def tearDown(self): while True: if self.job_id and self.job_id in self.client.list_jobs(): state = self.client.get_job(self.job_id).State if state in ['Waiting', 'Running']: self.client.stop_job(self.job_id) else: self.client.delete_job(self.job_id) else: break self.job_id = None def test_create_job(self): job_desc = self._get_job_desc() # Create jobs. job = self.client.create_job(job_desc) self.job_id = job.JobId self.assertTrue(self.job_id) def test_stop_job(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId state = self.client.get_job(job).State if state in ['Waiting', 'Running']: self.client.stop_job(job) self.assertRaises(ClientError, self.client.stop_job, job) def test_update_priority(self): old_priority = 100 new_priority = 200 job_desc = self._get_job_desc() job_desc.Priority = old_priority # Create jobs. job = self.client.create_job(job_desc) self.job_id = job.JobId status = self.client.get_job(job) self.assertEqual(status.Priority, old_priority) # update priority. self.assertRaises(ClientError, self.client.update_job_priority, job, new_priority) try: self.client.update_job_priority(job, new_priority) except ClientError as e: code = e.get_code() msg = e.get_msg() request_id = e.get_requestid() else: self.assertFalse(False, 'ClientError should be raised') status = self.client.get_job(job) if status.State in ['Waiting', 'Running']: self.client.stop_job(job) self.client.update_job_priority(job, new_priority) status = self.client.get_job(job) self.assertEqual(status.Priority, new_priority) def test_start_job(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId self.assertRaises(ClientError, self.client.start_job, job) status = self.client.get_job(job) if status.State in ['Waiting', 'Running']: self.client.stop_job(job) def test_list_images(self): image_list = self.client.list_images() for img in image_list: self.assertTrue(hasattr(img, 'ImageId')) self.assertTrue(not hasattr(img, 'ResourceId')) print(img.ImageId) def test_list_tasks(self): job_desc = self._get_job_desc() job = self.client.create_job(job_desc) self.job_id = job.JobId task_list = self.client.list_tasks(self.job_id) for task in task_list: print(task.TaskName) for instance in task.InstanceList: print(instance.InstanceId)