def check_status(emr, jobid, result_queue): while 1: status = emr.describe_jobflow(jobid) print status.state if "SHUTTING_DOWN" == status.state: result_queue.put("result!") break time.sleep(3)
def check_status(emr,jobid,result_queue): prevStatus = 'UNKNOWN' while 1: try: status = emr.describe_jobflow(jobid) if status.state == prevStatus: sys.stdout.write('.') sys.stdout.flush() else: print '\n'+status.state prevStatus = status.state if "SHUTTING_DOWN" == status.state: result_queue.put("result!") break; except EmrResponseError,e : print "exception happen",e.status, e.reason, e.body time.sleep(3)
step_args.append(params['data_region_code']) if params['data_variation']: step_args.append("--variation") step_args.append(params['data_variation']) step = JarStep(name='org.mozilla.up.TitleHarvester', jar='s3n://{0}/TitleHarvester.jar'.format(params['s3_bucket']), step_args=step_args ) print "instance groups: ", instance_groups emr = boto.connect_emr() job_id = emr.run_jobflow(name='TitleHarvester - {0}'.format(params['job_name']), availability_zone=params['availability_zone'], ec2_keyname=params['keypair'], log_uri='s3n://{0}/{1}-logs/'.format(params['s3_bucket'], params['job_name']), enable_debugging=False, keep_alive=params['keep_alive_mode'], ami_version='2.3.1', instance_groups=instance_groups, steps=[step], bootstrap_actions=[bootstrap_step1, bootstrap_step2] ) state = emr.describe_jobflow(job_id).state print "finished spawning job (note: starting still takes time)" print "job state = ", state print "job id = ", job_id