def setUp(self):
try:
Collection(collection_name).deregister()
except IndicoError:
pass
try:
Collection(alternate_name).deregister()
except IndicoError:
pass
try:
Collection(collection_name).clear()
except IndicoError:
pass
try:
Collection(image_collection_name).clear()
except IndicoError:
pass
try:
Collection(alternate_name).clear()
except IndicoError:
pass
def tearDown(self):
try:
Collection(collection_name).clear()
except IndicoError:
pass
try:
Collection(image_collection_name).clear()
except IndicoError:
pass
def test_rename(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
collection.rename(alternate_name)
new_collection = Collection(alternate_name)
# name no longer exists
with self.assertRaises(IndicoError):
collection = Collection(collection_name)
collection.train()
# collection is now accessible via the alternate name
new_collection.info()
new_collection.clear()
def train_custom_collection(data_file, collection_name):
data = json.load(open(data_file))
# Join a list of messages into a single example
data = map(_reformat_example, data)
batch_size = 20
c = Collection(collection_name, domain='topics')
try:
c.clear()
except indicoio.IndicoError:
# act like nothing happened
pass
for start_idx in tqdm(range(0, len(data), batch_size)):
messages = data[start_idx:start_idx + batch_size]
messages = filter(lambda x: x[0].strip() != "", messages)
c.add_data(messages)
c.train()
c.wait()
print "%s: %s" % (collection_name, str(c.info()))
return c
def _predict(self, conversation_id, assignee_id=None):
"""
Predict the assignee
"""
conversation = Conversation.find(id=conversation_id)
messages = [conversation.conversation_message
] + conversation.conversation_parts
for message in messages:
if SECRET_MENTION_ID in (message.body or ""):
# we've already suggested an Admin, don't suggest again
return
collection = Collection(COLLECTION_NAME, domain='topics')
text = extract_messages_text(messages)
results = collection.predict(text)
best_prediction = sorted(results.items(), key=itemgetter(1))[-1]
if best_prediction[1] > ASSIGNMENT_THRESHOLD:
if assignee_id is not None:
self._suggest_assignee(conversation, best_prediction[0])
else:
self._assign_assignee(conversation, best_prediction[0])
elif best_prediction[1] > SUGGESTION_THRESHOLD:
self._suggest_assignee(conversation, best_prediction[0])
else:
LOGGER.debug(
"Prediction did not produce high enough results for conversation {id} with {results}"
.format(id=conversation_id, results=results))
def _add_data_to_collection(self, conversation_id, assignee_id):
"""
Add data to the collection when a conversation gets assigned
"""
conversation = Conversation.find(id=conversation_id)
messages = [conversation.conversation_message
] + conversation.conversation_parts
for message in messages:
if SECRET_MENTION_ID in (message.body or ""):
# we've already suggested an Admin, don't suggest again
return
LOGGER.info("Adding data to collections for assignee {id}".format(
id=assignee_id))
messages = filter(lambda msg: not isinstance(msg.author, Admin),
messages)
message_text = extract_messages_text(messages)
collection = Collection(COLLECTION_NAME, domain='topics')
collection.add_data([message_text, assignee_id])
try:
collection.train()
except IndicoError as e:
pass
def test_add_large_batch(self):
collection = Collection(collection_name)
collection.add_data(test_data * 100)
collection.train()
collection.wait()
result = collection.predict(test_data[0][0])
assert test_data[0][1] in result.keys()
def main():
collection = Collection("clothes_collection_2")
# Clear any previous changes
'''
try:
collection.clear()
except:
pass
'''
train = generate_training_data("clothes_match_labeled_data_2.txt")
total = 0
for samples in train:
print("Adding {num} samples to collection".format(num=len(samples)))
collection.add_data(samples)
total += len(samples)
print(
"Added {total} samples to collection thus far".format(total=total))
collection.train()
collection.wait()
f = open('store1.pckl', 'wb')
pickle.dump(collection, f)
f.close()
def test_add_image_batch(self):
collection = Collection(image_collection_name)
collection.add_data(image_test_data)
collection.train()
collection.wait()
result = collection.predict(image_test_data[0][0])
assert image_test_data[0][1] in result.keys()
def test_clear_collection(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
assert collections()[collection_name]
collection.clear()
assert not collections().get(collection_name)
def test_clear_example(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
result = collection.predict(test_data[0][0])
assert test_data[0][1] in result.keys()
collection.remove_example(test_data[0][0])
collection.train()
collection.wait()
result = collection.predict(test_data[0][0])
assert test_data[0][1] not in result.keys()
def test_make_public(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
collection.register(make_public=True)
assert collection.info().get('registered')
assert collection.info().get('public')
collection.deregister()
assert not collection.info().get('registered')
assert not collection.info().get('public')
collection.clear()
def fit(self, X, y):
self.model = Collection("Enso-Sequence-Labeling-{}".format(str(hash(str(X) + str(y)))))
try:
self.model.clear()
except:
pass
X = list(zip(X, y))
for x in X:
self.model.add_data([x])
self.model.train()
self.model.wait()
def test_authorize_write_permissions(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
collection.register()
collection.authorize(email=test_user_email, permission_type='write')
assert test_user_email in collection.info().get('permissions').get('write')
assert not test_user_email in collection.info().get('permissions').get('read')
collection.deauthorize(email=test_user_email)
assert not test_user_email in collection.info().get('permissions').get('read')
assert not test_user_email in collection.info().get('permissions').get('write')
collection.clear()
def getRecommendation(subject, target_id):
collection = Collection("subject_collection_1")
msgToUser = '******'
pelajaran = subject.split(",")
recom_list = []
sentrec = []
# # Clear any previous changes
# try:
# collection.clear()
# except:
# pass
# train = generate_training_data("anandhibot/subject_match_labeled_data_1.txt")
# total = 0
# for samples in train:
# pushToUser(target_id, "training......")
# print("training.....")
# collection.add_data(samples)
# total += len(samples)
# pushToUser(target_id, "still training....")
# print("still training.....")
# collection.train()
# pushToUser(target_id, "selesai training")
# print("selesai training")
# collection.wait()
sort_key = itemgetter(1)
for sub in pelajaran:
recommendation = sorted(collection.predict(sub).items(), key=sort_key)
recom_list = recom_list + recommendation[-2:]
for rec in recom_list:
if rec[0] not in sentrec:
sentrec.append(rec[0])
msgToUser = "******" + ', '.join(
sentrec)
print("Message to user: "******"Request Timeout")
else:
pushToUser(target_id, msgToUser)
def _details(self, data_type, collection_name):
"""
Helper function
"""
coll = Collection(collection_name)
labels = []
try:
if data_type == "text":
labels = coll.predict("a").keys()
elif data_type == "image":
labels = coll.predict(np.array([[0]])).keys()
except IndicoError as e:
if e.message.endswith("does not exist.") or e.message.startswith(
"No trained model exists."):
pass
else:
raise ValueError(e.message)
return labels, coll
def main():
# Train Recommendation Model
collection = Collection("clothes_collection")
train = generate_training_data("clothes_match_labeled_data_2.txt")
total = 0
for samples in train:
print("Adding {num} samples to collection".format(num=len(samples)))
collection.add_data(samples)
total += len(samples)
print("Added {total} samples to collection thus far".format(total=total))
collection.train()
collection.wait()
return_list = main2()
f = open('store1.pckl','wb')
pickle.dump((collection,return_list),f)
f.close()
def test_collection(examples):
# Create Collection
collection = Collection("olin-slac-test-image-collection")
try:
collection.clear()
except:
pass
collection.add_data(examples)
collection.train()
# a blocking call until the collection/model is ready
collection.wait()
test_cat_image = "https://s-media-cache-ak0.pinimg.com/originals/84/71/e2/8471e2efdd2d3164895748ee8673124d.jpg"
print("Cat Test Result", collection.predict(test_cat_image))
test_dog_image = "https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcQgkJhtDW-n9qwGytWcYDYKq12AUPznwuQxWhgmxq0TFDLcYa95"
print("Dog Test Result", collection.predict(test_dog_image))
# Clear so we can use the same collection name during development
collection.clear()
def show_window_2(self):
# indico
indicoio.config.api_key = 'bb2ce29a2086bceb98bc297f6a961656'
img = cv.imread(self.path_entry.get())
collection = Collection("clothes_collection_1")
predict = collection.predict(img)
outcome = max(predict, key=predict.get)
outcome = outcome[5:]
# result img
result_im = Image.open("my_training_shirts/"+outcome+".jpg")
self.result_img = ImageTk.PhotoImage(result_im)
s_window = Toplevel()
s_window.geometry('600x400')
s_window.title('Result')
Label(s_window, text='your result is:').place(x=10,y=10)
Label(s_window, image=self.result_img).place(x=10,y=30)
Label(s_window, text='The link is:').place(x=200, y=10)
dic = get_link()
link = dic[outcome+'.jpg']
Label(s_window, text=link).place(x=200, y=30)
from indicoio.custom import Collection
indicoio.config.api_key = '4db161e811d460e7173d6dbb19384330'
collection = Collection("collection_name")
# Add Data
collection.add_data([["text1", "label1"], ["text2", "label2"], ...])
# Training
collection.train()
# Telling Collection to block until ready
collection.wait()
# Done! Start analyzing text
collection.predict("indico is so easy to use!")
def test_list_collection(self):
collection = Collection(collection_name)
collection.add_data(test_data)
collection.train()
collection.wait()
assert collections()[collection_name]
import scipy.misc
import glob
import indicoio
from indicoio.custom import Collection
# Configure API key for indico
indicoio.config.api_key = '3244a46ccac6ac4cf7d163a6240b5531'
# Connect to facial recognition collection
collection = Collection("facial_recognition three")
# Look for a face in the image given, localize it, save it, and predict whose face it belongs to
def testFaceImage(urlin, imgout):
face = indicoio.facial_localization(urlin, sensitivity=0.05, crop=True)
# print(face)
if len(face) >= 1:
face = face[0]
face = face['image']
scipy.misc.imsave(imgout, face)
prediction = collection.predict(imgout)
print(prediction)
max = 0
biggest = ""
for key, value in prediction.items():
if value > max:
biggest = key
def test_large_add_data(self):
collection = Collection(collection_name)
results = collection.add_data(test_data * 50, batch_size=50)
self.assertTrue(all([result is True for result in results]))
import csv
import indicoio
from indicoio.custom import Collection
# insert your API key
indicoio.config.api_key = "YOUR_API_KEY"
def clean_article(article):
return article.replace("\n",
" ").decode('cp1252').encode('utf-8', 'replace')
def test_model(test_list):
cleaned_test = [clean_article(text) for row in test_list for text in row]
print "Articles cleaned and ready for analysis!"
for data in cleaned_test:
print collection.predict(data)
if __name__ == "__main__":
# Replace "YOUR_COLLECTION_NAME" with the name you gave your dataset in CrowdLabel
collection = Collection("YOUR_COLLECTION_NAME")
with open('test_articles.csv', 'rU') as f:
test_list = csv.reader(f)
test_model(test_list)
import indicoio
from indicoio.custom import Collection
import secret
indicoio.config.api_key = secret.api_key
collection2 = Collection("colleection_name")
collection4 = Collection("colleeeection_name")
# Add Data
# collection.add_data([["indico is so easy", "label1"], ["text2", "label2"]])
# collection2.add_data([["https://upload.wikimedia.org/wikipedia/commons/2/27/Hillary_Clinton_official_Secretary_of_State_portrait_crop.jpg", "hillary"], ["http://i2.cdn.turner.com/money/dam/assets/160224112545-trump-nevada-victory-speech-780x439.jpg", "trump"]])
# collection3.add_data([["https://images-na.ssl-images-amazon.com/images/I/515LuXdi5ZL._SY355_.jpg", "Monet", "Impressionism"], ["http://destinationwilliamstown.org/wp-content/uploads/2015/01/Monet_Outbound.jpg", "Monet", "Impressionism"],["https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Claude_Monet_-_Meules,_milieu_du_jour.jpg/279px-Claude_Monet_-_Meules,_milieu_du_jour.jpg", "Monet", "Impressionism"], ["https://www.ibiblio.org/wm/paint/auth/monet/first/impression/impression.jpg", "Monet", "Impressionism"], ["http://www.kepeslap.com/images/22848/ren.jpg", "Renior", "Impressionism"], ["https://s-media-cache-ak0.pinimg.com/736x/5e/09/40/5e0940c7c67179aa515b4052ebe43d59.jpg", "Renior"], ["http://www.restaurant-renoir.de/upload/sonstige_bilder/galerie4g.jpg", "Renior", "Impressionism"], ["http://fr.wahooart.com/A55A04/w.nsf/O/BRUE-8EWPLG/$File/PIERRE-AUGUSTE-RENOIR-ALGIERS-THE-GARDEN-OF-ESSAI.JPG", "Renior", "Impressionism"]] )
collection4.add_data([
[
"https://images-na.ssl-images-amazon.com/images/I/515LuXdi5ZL._SY355_.jpg",
"Impressionism"
],
[
"http://destinationwilliamstown.org/wp-content/uploads/2015/01/Monet_Outbound.jpg",
"Impressionism"
],
[
"https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Claude_Monet_-_Meules,_milieu_du_jour.jpg/279px-Claude_Monet_-_Meules,_milieu_du_jour.jpg",
"Impressionism"
],
[
"https://www.ibiblio.org/wm/paint/auth/monet/first/impression/impression.jpg",
"Impressionism"
], ["http://www.kepeslap.com/images/22848/ren.jpg", "Impressionism"],
[
for line in f:
shirt, targets = line.split(":")
shirt_path = "training_shirts/{image}.jpg".format(
image=shirt.strip()
)
shirt_path = os.path.abspath(shirt_path)
# parse out the list of targets
target_list = targets.strip()[1:-1].split(",")
labels = map(lambda target: "label" + target.strip(), target_list)
yield [ (shirt_path, label) for label in labels]
raise StopIteration
if __name__ == "__main__":
collection = Collection("clothes_collection_1")
# Clear any previous changes
try:
collection.clear()
except:
pass
train = generate_training_data("clothes_match_labeled_data_1.txt")
total = 0
for samples in train:
print "Adding {num} samples to collection".format(num=len(samples))
collection.add_data(samples)
total += len(samples)
print "Added {total} samples to collection thus far".format(total=total)
def get(self, collection_name):
collection = Collection(collection_name, api_key=options.indico_key)
collection.train()
collection.wait()
origin_url = self.request.headers.get('Referer')
self.redirect('/' + '/'.join(origin_url.split('/')[3:]))
def get(self, collection_name):
Collection(collection_name, api_key=options.indico_key).clear()
self.redirect("/")
import indicoio
import operator
from indicoio.custom import Collection
# 1 anger
# 2 contempt
# 3 disgust
# 4 fear
# 5 happiness
# 6 neutral
# 7 sadness
# 8 surprise
indicoio.config.api_key = 'fcf8395d151536e2bcb06b64b44e94da'
collection = Collection("Selfie Caption Emotions_23")
collection.add_data([["0 0 0 0 0.83 0.3 0 0.01", "Be happy for this moment. This moment is your life."],
["0.01 0.02 0.1 0.12 0.9 0.4 0.01 0.01", "Be happy for this moment. This moment is your life."],
["0 0 0 0 1 0 0 0", "Being happy never goes out of style"],
["0 0.034 0.045 0.03 0.98 0.01 0.01 0.02", "Being happy never goes out of style"],
["0.01 0.032 0.035 0.04 1 0.08 0.02 0.05", "Being happy never goes out of style"],
["0 0 0 0 1 0 0 0.9", "YOLO!"],
["0.05 0.05 0.05 0.05 1 0.05 0.05 0.05", "LOL!"],
["0 0 0 0 1 0 0 0.9", "YOLO!"],
["0.05 0.05 0.05 0.05 1 0.05 0.05 0.05", "LOL!"],
["0 0 0 0 1 0.84 0.2 0.02", "Behind every successful man is a proud wife and a surprised mother-in-law"],
["0 0 0 0 1 0.92 0.2 0.02", "Behind every successful man is a proud wife and a surprised mother-in-law"],
["0 0 0 0.5 0 0 0 1", "Surprise!"],
["0 0 0 0.5 0 0 0 0.98", "Surprise!"],
["0 0.1 0.2 0.3 0.01 0 1 0", "Feeling Sad :("],
from flask import Flask, session, abort, redirect, url_for, escape, request, render_template
import indicoio
import json
from indicoio.custom import Collection
#API Setup
keyfile = open('static/indico-api-key.txt')
indicoio.config.api_key = keyfile.read()
keyfile.close()
#Global variables
wordList = open('static/bad_words.txt')
badWordList = wordList.read().split('\n')
wordList.close()
hCollection = Collection("HarrassmentCollection")
#Flask crap
application = Flask(__name__)
@application.route("/")
def hello():
return redirect(url_for('analyze'))
@application.route("/analyze", methods=['GET','POST'])
def analyze():
if request.method == 'POST':
if 'request-type' in request.form:
if request.form['request-type'] == 'web':
return render_template('analyze.html',results=analyzeText(request.form['comment-text']), commentText=request.form['comment-text'])