def perform(self, input):
print 'Printing ' + input
message = Serializer.parse(input, Message)
for sentence in message.body:
for token in sentence:
print token
print ''
def perform(self, input):
self.logger.info('Received results')
message = Serializer.parse(input, Message)
if message.type == 'command':
self.logger.debug("Received command message")
if message.body == 'DIE':
self.logger.info('Received the poison pill')
self.stop()
elif message.body.startswith('SAVE'):
self.logger.debug("Received the SAVE command.")
parts = message.body.split(' ')
if len(parts) > 1:
self.save(parts[1])
else:
self.save()
elif message.body == 'RESET':
self.logger.info("Clearing the question list.")
self.questions = list()
else:
self.logger.warn("Unknown command message: %s", message.body)
return
question = Question(message.body)
self.logger.debug("Received results for question %s", question.id)
self.questions.append(Question(message.body))
def perform(self, input):
print 'Tokenizing ' + input
message = Serializer.parse(input, Message)
tokenized_sentences = list()
for sentence in message.body:
tokenized = word_tokenize(sentence)
tokenized_sentences.append(tokenized)
message.body = tokenized_sentences
self.deliver(message)
def read_summary_questions(filepath):
with open(filepath, 'r') as fin:
dataset = Serializer.parse(fin, DataSet)
summary_questions = []
for question in dataset.questions:
if question.type == "summary":
summary_questions.append(question)
summary_questions = get_all_sentences(summary_questions)
return summary_questions
def perform(self, input):
message = Serializer.parse(input, Message)
if message.type == 'command':
self.logger.debug("Received command message")
if message.body == 'DIE':
self.logger.info('Received the poison pill')
self.stop()
return
else:
self.logger.warn("Unknown command message: %s", message.body)
message.body = self.getExpansions(message.body)
self.logger.debug('Delivering the message to next target')
self.deliver(message)
def perform(self, input):
message = Serializer.parse(input, Message)
if message.type == 'command':
self.logger.debug("Received command message")
if message.body == 'DIE':
self.logger.info('Received the poison pill')
self.stop()
return
else:
self.logger.warn("Unknown command message: %s", message.body)
question = Question(message.body)
question.exact_answer = self.tileSentences(question.ranked)
message.body = question
self.logger.debug('Delivering the message to next target')
self.deliver(message)
def _handler(self, channel, method, properties, message):
"""Default message handler that calls the user's `perform` method
and then acknowledges the message.
"""
message = Serializer.parse(message, Message)
if message.type == Message.COMMAND:
self.logger.debug("Received a command message")
if message.body == Message.POISON:
self.logger.info("Recieved the poison pill.")
self.stop()
else:
self.command(message.body)
else:
message.body = self.perform(message.body)
self.ack(method)
self.deliver(message)
def perform(self, input):
message = Serializer.parse(input, Message)
if message.type == 'command':
if message.body == 'DIE':
self.logger.info("Received command message DIE")
self.stop()
else:
self.logger.error('Unknown command message: %s', message.body)
self.deliver(message)
question = Question(message.body)
question.tokens = word_tokenize(question.body)
for snippet in question.snippets:
snippet.sentences = self.tokenize(snippet.text)
message.body = question
self.deliver(message)
def perform(self, input):
self.logger.info("Input received from route %s", self.route)
message = Serializer.parse(input, Message)
if message.type == 'command':
self.logger.debug("Received command message")
if message.body == 'DIE':
self.logger.info("Received the poison pill.")
self.stop()
else:
self.logger.warn("Unknown command message: %s", message.body)
else:
self.logger.info('Ranking sentences.')
self.logger.debug("Message body is a %s", str(type(message.body)))
question = Question(message.body)
question.ranked = self.getRankedList(question)
message.body = question
self.logger.debug('Delivering the message to next target')
self.deliver(message)
def read_original_data(data_file_name):
with open(os.path.join(DATA_PATH, data_file_name), "rb") as fp:
dataset = Serializer.parse(fp, DataSet)
summary_type_questions = []
for question in dataset.questions:
print "Question ID: " + question.id
print "Type: " + question.type
print "Body: " + question.body
print "Ideal answer: " + str(question.ideal_answer)
print "Exact answer: " + str(question.exact_answer)
if question.type == "summary":
summary_type_questions.append(question)
# print question.id + ': ' + question.body + ': ' + question.type
print 'Total questions: ', len(dataset.questions)
print 'Total summary-type questions: ', len(summary_type_questions)
return dataset
def perform(self, input):
""""""
""" Kaichen Chen 0304 """
print "Splitter {}: received, start to perform.".format(time())
message = Serializer.parse(input, Message)
if message.type == 'command':
if message.body == 'DIE':
self.logger.info("Received command message DIE")
self.stop()
else:
self.logger.error('Unknown command message: %s', message.body)
self.deliver(message)
question = Question(message.body)
question.tokens = word_tokenize(question.body)
for snippet in question.snippets:
snippet.sentences = self.tokenize(snippet.text)
message.body = question
self.deliver(message)
""" Kaichen Chen 0304 """
debug = True
# debug = False
if __name__ == '__main__':
if len(sys.argv) == 1:
print 'Usage: python pipeline.py <data.json>'
exit(1)
print 'pipeline.py: called, datasource: {}'.format(sys.argv[1])
# filename = 'data/training.json'
filename = sys.argv[1]
print 'Processing ' + filename
fp = open(filename, 'r')
dataset = Serializer.parse(fp, DataSet)
fp.close()
# The list of services to send the questions to.
""" Kaichen Chen 0304 """
if not debug:
pipeline = ['mmr.core', 'tiler.concat', 'results']
else:
pipeline = ['splitter', 'mmr.core', 'tiler.concat', 'results']
""" kaichenc 0307 """
host = os.environ.get('HOST', 'localhost')
print "DEBUG: Current host of Message Bus: {}".format(host)
count=0
def perform(self, input):
"""The input is expected to be a JSON string that can be parsed into a Message object"""
print 'Splitting ' + input
message = Serializer.parse(input, Message)
message.body = sent_tokenize(message.body)
self.deliver(message)
u'The increased odds ratios with African Americans was retained in post-menopausal women, while the protective odds ratios for pregnancy, smoking and oral contraceptives (OCs) became stronger in pre-menopausal women. The pattern by duration and timing of use does not suggest an etiologic role for OCs or hormone replacement therapy. ',
u'beginSection': u'abstract',
u'document': u'http://www.ncbi.nlm.nih.gov/pubmed/16570277',
u'endSection': u'abstract'
}, {
u'offsetInBeginSection': 1044,
u'offsetInEndSection': 1221,
u'text':
u'The use of hormone replacement therapy in symptomatic postmenopausal women either with previously treated disease or with dormant tumors is discussed, but remains controversial.',
u'beginSection': u'abstract',
u'document': u'http://www.ncbi.nlm.nih.gov/pubmed/15006250',
u'endSection': u'abstract'
}]
}
# question = Question(input)
# print question.body
message = Message()
message.route.append('results')
message.body = input
json = Serializer.to_pretty_json(message)
print json
message = Serializer.parse(json, Message)
print 'Route: ' + ', '.join(message.route)
question = Question(message.body)
print question.body
for snippet in question.snippets:
print snippet.text