def main(args): start = time.time() script_name, cluster_id, filename, file_id, num_nodes = args num_nodes = int(num_nodes) if not write_file_to_aws(filename, file_id): print "error i" return if not add_steps(cluster_id, file_id): print "error c" return data = wait_for_terminating(cluster_id) if (data == False): print "error w" return if not delete_key(file_id): print "error d" return if not run_ie(filename): print "error e" return end = time.time() total = end - start try: import billing billing = billing.Billing() billing.connect() billing.add_record(work_time_seconds=total, nodes=num_nodes, node_minute_price_cents=9, service='classification') billing.close() except Exception, e: pass
def process(self): try: print "Processing %s" % str(self.short_filename) self.read_old_jobflow_params() taskid = str(self.old_num + 1).strip() input_folder = 'input' + taskid self.write_file_to_aws(input_folder + "/" + self.short_filename) start_time = time.time() steps = self.get_steps(taskid, input_folder) if self.start_cluster: pass #self.start_hadoop_cluster(steps) else: pass #self.emr_connection.add_jobflow_steps(self.jobflow_id, steps) self.wait_for_terminating() secs = time.time() - start_time print secs, ' seconds elapsed' billing = billing.Billing() billing.connect() billing.add_record(work_time_seconds=secs, nodes=2, node_minute_price_cents=9, service='classification') billing.close() bucket = self.s3_connection.get_bucket(self.bucket_name) bucket.delete_key(input_folder + '/' + self.short_filename) run_ie(self.short_filename) except Exception, e: print "Unsupposed error: " + str(e)
