def check_status(emr, jobid, result_queue):
while 1:
status = emr.describe_jobflow(jobid)
print status.state
if "SHUTTING_DOWN" == status.state:
result_queue.put("result!")
break
time.sleep(3)
def check_status(emr,jobid,result_queue):
prevStatus = 'UNKNOWN'
while 1:
try:
status = emr.describe_jobflow(jobid)
if status.state == prevStatus:
sys.stdout.write('.')
sys.stdout.flush()
else:
print '\n'+status.state
prevStatus = status.state
if "SHUTTING_DOWN" == status.state:
result_queue.put("result!")
break;
except EmrResponseError,e :
print "exception happen",e.status, e.reason, e.body
time.sleep(3)
step_args.append(params['data_region_code'])
if params['data_variation']:
step_args.append("--variation")
step_args.append(params['data_variation'])
step = JarStep(name='org.mozilla.up.TitleHarvester',
jar='s3n://{0}/TitleHarvester.jar'.format(params['s3_bucket']),
step_args=step_args
)
print "instance groups: ", instance_groups
emr = boto.connect_emr()
job_id = emr.run_jobflow(name='TitleHarvester - {0}'.format(params['job_name']),
availability_zone=params['availability_zone'],
ec2_keyname=params['keypair'],
log_uri='s3n://{0}/{1}-logs/'.format(params['s3_bucket'], params['job_name']),
enable_debugging=False,
keep_alive=params['keep_alive_mode'],
ami_version='2.3.1',
instance_groups=instance_groups,
steps=[step],
bootstrap_actions=[bootstrap_step1, bootstrap_step2]
)
state = emr.describe_jobflow(job_id).state
print "finished spawning job (note: starting still takes time)"
print "job state = ", state
print "job id = ", job_id