def process_lines(self, line):
line = line.strip()
print line
#tokens = line.split()
#print tokens
if line.find("%") != -1:
tokens = line.split()
percent = tokens[len(tokens)-2]
now = datetime.datetime.now()
elapsed = now - self.jobstuff['startTime']
TCSendData.update_job_status(self.jobstuff['name'], str(elapsed), percent )
if line.find("MR plan size after optimization:") != -1:
tokens = line.split()
self.jobstuff['nodes'] = tokens[len(tokens)-1]
if line.find("totalInputFileSize") != -1:
tokens = line.split()
self.jobstuff['size'] = tokens[len(tokens)-1].split("=")[1]
if (line.find("org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher") != -1) and not self.jobstuff['started']:
self.jobstuff['started'] = True
TCSendData.send_new_job(self.jobstuff['name'],self.jobstuff['site'],self.jobstuff['startTimeStr'],self.jobstuff['nodes'],self.jobstuff['size'], self.jobstuff['description'])
if ((line.find("Success") != -1) and (self.jobstuff['finished'] == False)):
self.jobstuff['finished'] = True
now = datetime.datetime.now()
elapsed = now - self.jobstuff['startTime']
TCSendData.finish_job(self.jobstuff['name'], str(elapsed) )
import TCSendData
output = ""
first = True
file = open('/home/hadoop/geni/geni-process-results','r')
for line in file:
if first == True:
first = False
else:
output += ","
line_split = line.split('\t')
output += line_split[0] + ":" + line_split[1].strip()
print output
params = {'name': 'Processing1B', 'entries':output}
TCSendData.http_send('/jobs/api/add_batch_hadoop_result/', params, 'pigLog.log')
def sendPigResults(jobName, entryList):
entryString = ','.join(entryList)
params = {'name': jobName, 'entries':entryString}
TCSendData.http_send('/jobs/api/add_batch_hadoop_result/', params, 'pigLog.log')