def __init__(self):
memcached_addr = os.environ.get('MEMCACHED_PORT_11211_TCP_ADDR')
if memcached_addr:
log('Memcached: ' + memcached_addr)
self.client = Client([(memcached_addr, 11211)])
else:
log('Memcached: localhost')
self.client = Client([('127.0.0.1', 11211)])
def __init__(self):
mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR')
if mongodb_addr:
log('MongoDB: ' + mongodb_addr)
self.db = MongoClient(mongodb_addr, 27017).lucida
else:
log('MongoDB: localhost')
self.db = MongoClient().lucida
self.users = self.db.users
def learn_text(self, LUCID, text_type, text_data, text_id):
for service in config.Service.LEARNERS['text']: # add concurrency?
knowledge_input = self.create_query_input(text_type, text_data,
[text_id])
client, transport = self.get_client_transport(service)
log('Sending learn_text request to QA')
client.learn(
str(LUCID),
self.create_query_spec('knowledge', [knowledge_input]))
transport.close()
def learn_image(self, LUCID, image_type, image_data, image_id):
for service in config.Service.LEARNERS['image']: # add concurrency?
knowledge_input = self.create_query_input(image_type, image_data,
[image_id])
client, transport = self.get_client_transport(service)
log('Sending learn_image request to IMM')
client.learn(
str(LUCID),
self.create_query_spec('knowledge', [knowledge_input]))
transport.close()
def infer(self, LUCID, service_graph, text_data, image_data):
# Create the list of QueryInput.
query_input_list = []
for node in service_graph.node_list:
service = self.SERVICES[node.service_name]
data = text_data if service.input_type == 'text' else image_data
host, port = service.get_host_port()
tag_list = [host, str(port), str(len(node.to_indices))]
for to_index in node.to_indices:
tag_list.append(str(to_index))
query_input_list.append(
self.create_query_input(service.input_type, data, tag_list))
query_spec = self.create_query_spec('query', query_input_list)
# Go through all starting indices and send requests.
result = []
for start_index in service_graph.starting_indices:
service = self.SERVICES[service_graph.get_node(
start_index).service_name]
client, transport = self.get_client_transport(service)
log('Sending infer request to ' + service.name)
result.append(client.infer(str(LUCID), query_spec))
transport.close()
return ' '.join(result)
def __init__(self, SERVICES):
self.SERVICES = SERVICES
log('Pre-configured services: ' + str(SERVICES))
def learn_route():
options = {}
username = session['username']
try:
form = request.form
# Deal with POST requests.
if request.method == 'POST':
# If the request does not contain an "op" field.
if not 'op' in request.form:
raise RuntimeError('Did you click the button?')
# Add image knowledge.
elif form['op'] == 'add_image':
image_type = 'image'
label = form['label']
# Check the uploaded image.
upload_file = request.files['file']
if upload_file.filename == '':
raise RuntimeError('Empty file is not allowed')
check_image_extension(upload_file)
# Check the label of the image.
check_text_input(label)
# Check whether the user can add one more image.
database.check_add_image(username)
# Generate the id.
image_data = upload_file.read()
image_id = hashlib.md5(
username + str(datetime.datetime.now())).hexdigest()
# Send the image to IMM.
upload_file.close()
thrift_client.learn_image(username, image_type, image_data,
image_id)
# Add the image into the database.
database.add_image(username, image_data, label, image_id)
# Delete image knowledge.
elif form['op'] == 'delete_image':
image_type = 'unlearn'
image_id = form['image_id']
# Send the unlearn request to IMM.
thrift_client.learn_image(username, image_type, '', image_id)
# Delete the image from the database.
database.delete_image(username, image_id)
# Add text knowledge.
elif form['op'] == 'add_text' or form['op'] == 'add_url':
text_type = 'text' if form['op'] == 'add_text' else 'url'
text_data = form['knowledge']
# Check the text knowledge.
check_text_input(text_data)
# Check whether the user can add one more piece of text.
database.check_add_text(username)
# Generate the id.
text_id = hashlib.md5(
username + text_data +
str(datetime.datetime.now())).hexdigest()
# Send the text to QA.
thrift_client.learn_text(username, text_type, text_data,
text_id)
# Add the text knowledge into the database.
database.add_text(username, text_type, text_data, text_id)
# Delete text knowledge.
elif form['op'] == 'delete_text':
text_type = 'unlearn'
text_id = form['text_id']
# Send the unlearn request to QA.
thrift_client.learn_text(username, text_type, '', text_id)
# Delete the text from into the database.
database.delete_text(username, text_id)
else:
raise RuntimeError('Did you click the button?')
except Exception as e:
log(e)
if str(e) == 'TSocket read 0 bytes':
e = 'Back-end service encountered a problem'
options['error'] = e
try:
# Retrieve knowledge.
options['pictures'] = database.get_images(username)
options['text'] = database.get_text(username)
except Exception as e:
log(e)
options['error'] = e
return render_template('learn.html', **options)
def count_images(self, username):
log('Retrieving the number of images from images_' + username)
return self.get_image_collection(username).count()
def get_images(self, username):
log('Retrieving all images from images_' + username)
# Notice image['data'] was encoded using Base64.
return [image for image in self.get_image_collection(username).find()]
def get_text(self, username):
log('Retrieving text from text_' + username)
return [text for text in self.get_text_collection(username).find()]