while numSucceededJobs < number_execs:
try:
connection = getConnection(user, password, project_name, project_id, main_ip)
input_size = DEF_INPUT_SIZE_MB
num_reduces = mapred_reduce_tasks
if job['name'] in ['Pi Estimator']:
num_reduces = '1'
input_size = job['args'][0]
job_res = connection['sahara'].runJavaActionJob(main_class=job['main_class'], job_id=job['template_id'], cluster_id=cluster_id, reduces=num_reduces, args=job['args'])
saveJobResult(job_res, job['name'], cluster_size, master_ip, num_reduces, job_number, output_file, input_size)
if (job_res['status'] != 'SUCCEEDED'): numFailedJobs += 1
else: numSucceededJobs += 1
deleteHDFSFolder(private_keypair_path,master_ip)
print "Break time... go take a coffee and relax!"
time.sleep(5)
except Exception, e:
print "Exception: ", e
connection = getConnection(user, password, project_name, project_id, main_ip)
deleteHDFSFolder(private_keypair_path,master_ip)
print 'FINISHED FOR CLUSTER ' + cluster_name
print 'PLEASE DELETE CLUSTER IF YOU RE NOT GONNA USE IT ANYMORE :D'
sendMail('Success!', gmail_user + '@gmail.com', gmail_user + '@gmail.com', gmail_password, cluster_size, output_file)
while numSucceededJobs < number_execs:
try:
input_size = DEF_INPUT_SIZE_MB
num_reduces = mapred_reduce_tasks
if job['name'] in ['Pi Estimator']:
num_reduces = '1'
input_size = job['args'][0]
job_res = connection['sahara'].runJavaActionJob(main_class=job['main_class'], job_id=job['template_id'], cluster_id=cluster_id, reduces=num_reduces, args=job['args'])
saveJobResult(job_res, job['name'], cluster_size, master_ip, num_reduces, job_number, output_file, input_size)
if (job_res['status'] != 'SUCCEEDED'):
numFailedJobs += 1
else:
numSucceededJobs += 1
deleteHDFSFolder(private_keypair_path,master_ip)
print "Break time... go take a coffee and relax!"
time.sleep(5)
except Exception, e:
print "Exception: ", e
connection = getConnection(user, password, project_name, project_id, main_ip)
deleteHDFSFolder(private_keypair_path,master_ip)
sendMail('Finished job %s\nsucces_jobs:%s failed_jobs:%s reduce fator %s' % (job['name'], numSucceededJobs, numFailedJobs, str(mapred_factor)) , gmail_user + '@gmail.com', gmail_user + '@gmail.com', gmail_password, cluster_size, output_file)
connection['sahara'].deleteCluster(cluster_id)
print 'FINISHED FOR CLUSTER ' + cluster_name
sendMail('Success!', gmail_user + '@gmail.com', gmail_user + '@gmail.com', gmail_password, cluster_size, output_file)
