コード例 #1
0
ファイル: process_file.py プロジェクト: valeter/nlp-site
def main(args):
	start = time.time()

	script_name, cluster_id, filename, file_id, num_nodes = args
	num_nodes = int(num_nodes)

	if not write_file_to_aws(filename, file_id):
		print "error i"
		return

	if not add_steps(cluster_id, file_id):
		print "error c"
		return
	
	data = wait_for_terminating(cluster_id)
	if (data == False):
		print "error w"
		return

	if not delete_key(file_id):
		print "error d"
		return

	if not run_ie(filename):
		print "error e"
		return

	end = time.time()
	total = end - start

	try:
		import billing
		billing = billing.Billing()
		billing.connect()
		billing.add_record(work_time_seconds=total, nodes=num_nodes, node_minute_price_cents=9, service='classification')
		billing.close()
	except Exception, e:
		pass
コード例 #2
0
ファイル: make_result_file.py プロジェクト: valeter/nlp-site
	def process(self):
		try:
			print "Processing %s" % str(self.short_filename)
			
			self.read_old_jobflow_params()

			taskid = str(self.old_num + 1).strip()
			input_folder = 'input' + taskid
			self.write_file_to_aws(input_folder + "/" + self.short_filename)
			
			start_time = time.time()

			steps = self.get_steps(taskid, input_folder)
			if self.start_cluster:
				pass
				#self.start_hadoop_cluster(steps)
			else:
				pass
				#self.emr_connection.add_jobflow_steps(self.jobflow_id, steps)

			self.wait_for_terminating()

			secs = time.time() - start_time
			print secs, ' seconds elapsed'
			
			billing = billing.Billing()
			billing.connect()
			billing.add_record(work_time_seconds=secs, nodes=2, node_minute_price_cents=9, service='classification')
			billing.close()
			
			bucket = self.s3_connection.get_bucket(self.bucket_name)
			bucket.delete_key(input_folder + '/' + self.short_filename)
			
			run_ie(self.short_filename)
		except Exception, e:
			print "Unsupposed error: " + str(e)