def engine(username):
favoriate_tools = my_redis.get_user_tool(username)
role_tools = my_redis.get_user_tool('h'+username)
engine_client = predictionio.EngineClient(url="http://167.99.183.243:8000")
recommended_tool_properties = []
favoriate_tool_properties = []
role_tool_properties = []
tool_name = []
recommended_tool_ids = []
for num in range(0,len(favoriate_tools)):
current_favoriate = my_redis.get_tool_properties(favoriate_tools[num])
favoriate_tool_properties.append(current_favoriate)
current_favoriate_name = current_favoriate[2]
print favoriate_tool_properties, current_favoriate_name
temp = ''
if num == 0:
temp = engine_client.send_query({"items": [favoriate_tools[num]], "num": 2})
recommended_tool_ids.append([current_favoriate_name, temp['itemScores'][0]['item']])
recommended_tool_ids.append([current_favoriate_name, temp['itemScores'][1]['item']])
else:
temp = engine_client.send_query({"items": [favoriate_tools[num]], "num": 4})
temp_id = [item[1] for item in recommended_tool_ids]
for item in temp['itemScores']:
if item['item'] not in temp_id:
print '%s not in %s' %(item['item'], temp_id)
recommended_tool_ids.append([current_favoriate_name, item['item']])
break
for item in recommended_tool_ids:
recommended_tool_properties.append([item[0],my_redis.get_tool_properties(item[1])])
for tool in role_tools:
role_tool_properties.append(my_redis.get_tool_properties(tool))
tools = my_redis.get_tool_properties('tools')
for tool in tools:
name = my_redis.get_tool_properties(tool)[2]
if '_' in name:
name = name.replace('_',' ')
tool_name.append(name)
engine_client = predictionio.EngineClient(url="http://172.17.0.2:8001")
hit_tools = []
hits = engine_client.send_query({})
temp = hits['itemScores']
for num in range(0,6):
if temp[num]['item'] != 'Surface':
hit_tools.append([num, my_redis.get_tool_properties(temp[num]['item'])[2], my_redis.get_tool_properties(temp[num]['item'])[1]])
print "hit_tools %s" %hit_tools
# print 'favoriate_tool_properties%s'%favoriate_tool_properties
return render_template('engine.html', username = username, favoriate_tools = favoriate_tool_properties, recommended_tool = recommended_tool_properties, role_tool = role_tool_properties, tool_name = tool_name, hit_tools=hit_tools,error = error)
def engine():
name = request.form.get('names')
print name
engine_client = predictionio.EngineClient(url="http://167.99.183.243:8000")
r = engine_client.send_query({"items": ["i1"], "num": 3})
return render_template('engine1.html', customers=r, error=error)
def _send_query(events_list):
"""
Queries PredictionIO Engine for items
:param events_list: Items which are liked by the user
:return: Event ids recommended by the engine
"""
pio_engine = getattr(settings, 'PIO_ENGINE_ENDPOINT', None)
if not pio_engine:
return []
engine_client = predictionio.EngineClient(url=pio_engine)
items_set = set()
categories_set = set()
for event in events_list:
items_set.add(event.id)
categories_set = categories_set.union(
set(event.category.values_list('name', flat=True)))
try:
result = engine_client.send_query({
"items": list(items_set),
"num": 10,
"categories": list(categories_set)
})
engine_client.close()
except predictionio.NotFoundError:
print "Recommendation engine not found"
return []
events = result.get('itemScores', [])
return [e['item'] for e in events]
def export_events(client):
print client.get_status()
print "Exporting data..."
engine_client = predictionio.EngineClient(url="http://52.8.31.209:8888")
data = engine_client.send_query({"user": "******", "num": 4})
#pprint(data)
value_score = []
value_item = []
incr = 0
#strvalue=json.dumps(data,separators=(',',':'))
#print strvalue
myDbConn = MySQLdb.Connect(host='127.0.0.1',
user='******',
passwd='nathiya',
db='predictionio')
cur = myDbConn.cursor()
for key in data.keys():
strvalue = (data[key])
print strvalue
for i in range(len(strvalue)):
print strvalue[i]
for k in strvalue[i].keys():
item = strvalue[k]
score = strvalue[k + 1]
print item, score
''' if (k=='item'):
def export_events(client):
print client.get_status()
print "Exporting data..."
user_id = raw_input("enter the customer_id")
num = int(raw_input("Enter the number of recommendations"))
engine_client = predictionio.EngineClient(url="http://52.8.31.209:8888")
data = engine_client.send_query({"user": user_id, "num": num})
myDbConn = MySQLdb.Connect(host='127.0.0.1',
user='******',
passwd='nathiya',
db='predictionio')
cur = myDbConn.cursor()
for key in data.keys():
strvalue = (data[key])
print strvalue
for i in range(len(strvalue)):
print strvalue[i]
for k, v in strvalue[i].items():
print k, v
if (k == 'item'):
event_item = v
else:
event_score = v
user_id = 1
add_event = (
'INSERT INTO eventdata(user_id,recom_item_id,score)VALUES(%s,%s,%s)'
)
cur.execute(add_event, (user_id, event_item, event_score))
myDbConn.commit()
myDbConn.close()
print "Done"
def run_map_test(data, eventNames, users=None, primaryEvent=cfg.testing.primary_event,
consider_non_zero_scores=cfg.testing.consider_non_zero_scores_only,
num=200, K=cfg.testing.map_k, test=False, predictionio_url="http://0.0.0.0:8000"):
N_TEST = 2000
d = {}
res_data = {}
engine_client = predictionio.EngineClient(url=predictionio_url)
for rec in data:
if rec.event == primaryEvent:
user = rec.entityId
item = rec.targetEntityId
if (users is None) or (user in users):
d.setdefault(user, []).append(item)
if test:
holdoutUsers = d.keys()[1:N_TEST]
else:
holdoutUsers = d.keys()
prediction = []
ground_truth = []
user_items_cnt = 0.0
users_cnt = 0
for user in tqdm(holdoutUsers):
q = {
"user": user,
"eventNames": eventNames,
"num": num,
}
try:
res = engine_client.send_query(q)
# Sort by score then by item name
tuples = sorted([(r["score"], r["item"]) for r in res["itemScores"]], reverse=True)
scores = [score for score, item in tuples]
items = [item for score, item in tuples]
res_data[user] = {
"items": items,
"scores": scores,
}
# Consider only non-zero scores
if consider_non_zero_scores:
if len(scores) > 0 and scores[0] != 0.0:
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
else:
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
except predictionio.NotFoundError:
print("Error with user: %s" % user)
return ([metrics.mapk(ground_truth, prediction, k) for k in range(1, K + 1)],
res_data, user_items_cnt / (users_cnt + 0.00001))
def recommend(request):
if 'user' in request.session:
user = request.session['user']
length = 38
#Recommend Film
check_api = False
try:
urllib2.urlopen('http://localhost:8000', timeout=1)
check_api = True
except urllib2.URLError as err:
check_api = False
r = 0
if check_api:
engine_client = predictionio.EngineClient(
url="http://localhost:8000")
r = engine_client.send_query({
"user":
user,
"num":
length,
"blackList":
request.session['blacklist']
})
m = json.dumps(r)
js = json.loads(m)
movieIds = []
movielensIds = []
#Read file csv
if js:
for item in js['itemScores']:
print "item: ", item['item'], "\t score: ", item['score']
with open('links.csv') as file:
reader = csv.reader(file)
for row in reader:
if row[0] == item['item']:
# with open("backlist.dat") as fileb:
# bls = file.readlines()
# for bl in bls:
# if bl != row[0]:
movieIds.append(row[2])
movielensIds.append(row[0])
loops = []
for item in range(1, len(movieIds) + 1):
loops.append(item)
return render(
request, 'recommend.html', {
'movieIds': json.dumps(movieIds),
'length': loops,
'movielensIds': json.dumps(movielensIds),
'user': user
})
def main():
parser = argparse.ArgumentParser(description="Ask for similar objects")
parser.add_argument('--items', default='1,10')
parser.add_argument('--num', default='5')
parser.add_argument('--url', default="http://localhost:8000")
parser.add_argument('--minitemid', default='0')
args = parser.parse_args()
print(args)
engine_client = predictionio.EngineClient(url=args.url)
print engine_client.send_query({
"items": args.items.split(","),
"num": int(args.num),
"minItemID": args.minitemid
})
def run(self):
PATH = "/Users/Hamza07/Desktop/rec_apps/"
with open(osp.join(PATH, self.options['<app_name>'], 'access_key'),
'r') as myfile:
data = myfile.read()
accessKey = data.replace('\n', '')
engine_client = predictionio.EngineClient(
url='http://localhost:8000', )
result = engine_client.send_query(data={
"uid": self.options['<user_id>'],
"n": self.options['<num_recs>']
})
# print(request)
# try:
# result = request.get_response() # check the request status and get the return data.
print(result)
num_friends_in_common = get_num_friends_in_common(friends_friend)
if num_friends_in_common > 0:
add_friend(friends_friend, num_friends_in_common)
num_ff_recs += 1
if num_ff_recs >= 50:
break
##################################
# add via prediction io
if Configuration.PREDICTION_IO_ENABLED and len(
suggested) < 50 and not is_limited('last_pio_check', ttl=60):
try:
# flesh out the rest via prediction io
engine_client = predictionio.EngineClient(
url='http://{}:{}'.format(Configuration.PREDICTION_IO_HOST,
Configuration.PREDICTION_IO_PORT))
predictions = engine_client.send_query({
'user': str(user_id),
'num': 50,
'blackList': [str(user_id)]
})
for r in predictions.get('itemScores'):
suggest_id = int(r['item'])
if should_suggest(suggest_id):
add_friend(suggest_id)
if len(suggested) >= num_friends_to_recommend:
break
import predictionio
from dotenv import load_dotenv
import os
if __name__ == '__main__':
load_dotenv()
engine_client = predictionio.EngineClient(url='http://{}:{}'.format(
os.environ['ENGINE_HOST'], os.environ['ENGINE_PORT']))
print(engine_client.send_query({"attr0": 54, "attr1": 39, "attr2": 34}))
def run_map_test(
data,
eventNames,
users=None,
primaryEvent=cfg.testing.primary_event,
consider_non_zero_scores=cfg.testing.consider_non_zero_scores_only,
num=20,
K=cfg.testing.map_k,
test=False,
predictionio_url="http://0.0.0.0:8000"):
N_TEST = 2000
d = {}
res_data = {}
engine_client = predictionio.EngineClient(url=predictionio_url)
if (not users is None):
users = set(users)
for rec in data:
if rec.event == primaryEvent:
user = rec.entityId
item = rec.targetEntityId
if (users is None) or (user in users):
d.setdefault(user, []).append(item)
if test:
holdoutUsers = d.keys()[1:N_TEST]
else:
holdoutUsers = d.keys()
prediction = []
ground_truth = []
user_items_cnt = 0.0
users_cnt = 0
FNULL = open(os.devnull, 'w')
with open("batchpredict-input.json", "w") as queryFile:
for user in tqdm(holdoutUsers):
q = {
"user": user.encode('utf-8'),
"eventNames": eventNames,
"num": num,
}
qString = str(json.dumps(q)).strip("\n").replace("'", "\"") + "\n"
# print("QUERY" + qString)
# sys.stdout.flush()
queryFile.write(qString)
# queryFile.close()
call(["pio", "batchpredict", "-- --master local[*] --driver-memory 28g"],
stdout=FNULL,
stderr=subprocess.STDOUT)
#call("pio batchpredict", shell=True)
FNULL.close()
with open("batchpredict-output.json", "r") as responsesFile:
lines = responsesFile.read().splitlines()
for line in lines:
try:
fullRes = json.loads(line)
res = fullRes['prediction']
# print("RES " + str(fullRes))
# sys.stdout.flush()
user = fullRes['query']['user']
# Sort by score then by item name
tuples = sorted([(r["score"], r["item"])
for r in res["itemScores"]],
reverse=True)
scores = [score for score, item in tuples]
items = [item for score, item in tuples]
res_data[user] = {
"items": items,
"scores": scores,
}
# Consider only non-zero scores
if consider_non_zero_scores:
if len(scores) > 0 and scores[0] != 0.0:
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
else:
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
except predictionio.NotFoundError:
print("Error with user: %s" % user)
return ([
metrics.mapk(ground_truth, prediction, k) for k in range(1, K + 1)
], res_data, user_items_cnt / (users_cnt + 0.00001))
import predictionio
engine_client = \
predictionio.EngineClient(url="http://localhost:8000", timeout=20)
print engine_client.send_query({"word": "goose"})
engine_client.close()
def index(request):
if 'user' not in request.session:
return HttpResponseRedirect('/polls/signout')
#return HttpResponseRedirect('/polls')
user = request.session['user']
#print user
mostpopular = 12
lengthMostPopular = []
for item in range(1, mostpopular + 1):
lengthMostPopular.append(item)
length = 36
check_api = False
try:
urllib2.urlopen('http://localhost:8000', timeout=1)
check_api = True
except urllib2.URLError as err:
check_api = False
r = 0
if check_api:
engine_client = predictionio.EngineClient(url="http://localhost:8000")
r = engine_client.send_query({
"user": user,
"num": length,
"blackList": request.session['blacklist']
})
m = json.dumps(r)
js = json.loads(m)
loops = []
#for index in range(0:3):
movieIds = []
movielensIds = []
#Write file Watched
user = request.session['user']
print user
# if request.session['mid'] is not None:
# watchedFilm = request.session['mid']
# print watchedFilm
# watchedFile = open(user+'watched.txt','a')
# print "da ghi"
# watchedFile.write(watchedFilm)
# watchedFile.write('\n')
# watchedFile.close()
movies = []
movies.append('0')
#Read file Watched
#Read file csv
if js:
for item in js['itemScores']:
print "item: ", item['item'], "\t score:", item['score']
with open('links.csv') as file:
reader = csv.reader(file)
for row in reader:
if row[0] == item['item']:
movieIds.append(row[2])
movielensIds.append(row[0])
print movieIds
#Remove duplicate of moviewIds
# newlistMoviewIds = []
# for i in movieIds:
# if i not in movies:
# newlistMoviewIds.append(i)
# print newlistMoviewIds
#Remove duplicate of movielensIds
# newlistMovieLensIds = []
# for i in movieIds:
# if i not in newlistMovieLensIds:
# newlistMovieLensIds.append(i)
for item in range(1, 13):
loops.append(item)
return render(
request, 'index.html', {
'movieIds': json.dumps(movieIds),
'movielensIds': json.dumps(movielensIds),
'length': loops,
'lengthMostPopular': lengthMostPopular,
'user': user
})
def __init__(self, url, data_source):
assert isinstance(data_source, DataSource)
self.client = predictionio.EngineClient(url)
self.data_source = data_source
print(zero)
print(uno)
print(due)
file_data = open("data.txt", "w")
for i in range(len(X_train)):
file_data.write(f"{y_train.iloc[i]}" + ",")
for val in X_train.iloc[i]:
file_data.write(f"{val}" + " ")
file_data.write("\n")
file_data.close()
test_feat = {}
test_lab = {}
engine_client = predictionio.EngineClient(url="http://192.168.1.132:8000")
diversi = 0
uguali = 0
for i in range(len(X_test)):
count = 0
for val in X_test.iloc[i]:
test_feat[X_test.iloc[i].index[count]] = val
print(test_feat)
count += 1
y_pred = engine_client.send_query(test_feat)
print(y_pred)
if float(y_pred['status']) != float(y_test.iloc[i]):
diversi += 1
else:
uguali += 1
print(uguali)
def __init__(self, url=None):
self.db = Database()
self.engine_client_class = predictionio.EngineClient(url="http://192.168.1.127:8000")
self.engine_client_rec = predictionio.EngineClient(url="http://192.168.1.127:8001")
self.event_client = predictionio.EventClient(access_key="6S_JIW0eesru9ea5NsTwO0dIBea-Q8Wp2tBX7Kh8EojF4KRme8JTnqpr5J0M1hTk", url="http://192.168.1.127:7070", threads=5, qsize=500)
label1 += 1
print(count)
print('done')
print('------------RESULT------------')
print('Test Count: %d' % count)
print('Label 0 : %d(%f%%)' % (label0, label0 / count * 100))
print('Label 1 : %d(%f%%)' % (label1, label1 / count * 100))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Format data.")
parser.add_argument('--url', default="http://localhost:8000")
parser.add_argument('--file', default=None)
parser.add_argument('--out', default=None)
parser.add_argument('--val', default="100")
parser.add_argument('--verbose', default="0")
args = parser.parse_args()
print(args)
if args.file is None or args.out is None:
parser.print_help()
exit()
with open(args.file) as filein, open(args.out, 'w') as fileout:
client = predictionio.EngineClient(url=args.url)
test(client,
filein,
fileout,
verbose=args.verbose,
val=float(args.val))
def run_map_test(
data,
eventNames,
users=None,
primaryEvent=cfg.testing.primary_event,
consider_non_zero_scores=cfg.testing.consider_non_zero_scores_only,
num=200,
K=cfg.testing.map_k,
test=False,
predictionio_url="http://0.0.0.0:8000"):
N_TEST = 2000
d = {}
res_data = {}
engine_client = predictionio.EngineClient(url=predictionio_url)
for rec in data:
if rec.event == primaryEvent:
user = rec.entityId
item = rec.targetEntityId
if not users or user in users:
d.setdefault(user, []).append(item)
if test:
holdoutUsers = [*d.keys()][1:N_TEST]
else:
holdoutUsers = [*d.keys()]
prediction = []
ground_truth = []
user_items_cnt = 0.0
users_cnt = 0
for user in tqdm(holdoutUsers):
q = {"user": user, "eventNames": eventNames[0], "num": num}
try:
res = engine_client.send_query(q)
# Sort by score then by item name
tuples = sorted([(r["score"], r["item"])
for r in res["itemScores"]],
reverse=True)
scores = [score for score, item in tuples]
items = [item for score, item in tuples]
res_data[user] = {
"items": items,
"scores": scores,
}
# Consider only non-zero scores
if consider_non_zero_scores:
if len(scores) > 0 and scores[0] != 0.0:
with open("logs.txt", "a") as log_file:
log_file.write("\n Successfully got scores")
log_file.write("\n user: "******"\n eventNames: " + str(eventNames))
# logging.debug("consider_non_zero_scores, eventNames %s" % eventNames)
if "mm-" in str(eventNames) or "show-hold" in str(
eventNames) or "offer-hold" in str(eventNames):
with open("logs.txt", "a") as log_file:
log_file.write(
"\n One of 3 signal eventName has nonzero score"
+ str(eventNames))
log_file.write("\n Scores were: " + str(scores))
log_file.write("\n items: " + str(items))
# logging.debug("mm- eventName has nonzero score %s" % eventNames)
# logging.debug("Scores were %s" % scores)
# logging.debug("items: %s" % items)
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
else:
with open("logs.txt", "a") as log_file:
log_file.write("\n")
log_file.write("Event not included in map@k: " +
str(eventNames))
log_file.write("\n Scores were: " + str(scores))
# logging.debug("Consider non zero, scores were zero: ")
# logging.debug("Scores were %s" % scores)
# logging.debug("consider_non_zero_scores, eventNames %s" % eventNames)
else:
prediction.append(items)
ground_truth.append(d.get(user, []))
user_items_cnt += len(d.get(user, []))
users_cnt += 1
with open("logs.txt", "a") as log_file:
log_file.write("\n End of try for qs: ")
log_file.write("\n user: "******"\n eventNames: " + str(eventNames[0]))
log_file.write("\n End of try, Scores were: " + str(scores))
except predictionio.NotFoundError:
# logging.debug("Error: %s " % predictionio.NotFoundError)
# logging.debug("Error with user: %s" % user)
print("Error with user: %s" % user)
except Exception as e:
# logging.debug("General exception: %s" % e)
with open("logs.txt", "a") as log_file:
log_file.write("\n General exception: " + str(e))
# logging.debug("ground_truth after user added %s: " % ground_truth)
# logging.debug("event_group: %s" % eventNames)
map_l = [
metrics.mapk(ground_truth, prediction, k) for k in range(1, K + 1)
], res_data, user_items_cnt / (users_cnt + 0.00001)
# logging.debug("Map@k calculation list: %s" % map_l)
return (map_l)
"""
Send sample query to prediction engine
"""
import predictionio
engine_client = predictionio.EngineClient(url="http://localhost:8000")
event_client = predictionio.EventClient(
access_key=
'Zwybqp8kKTdTApM0S53HEB31Ti6bgiaDFGNMZ8TlCyfhJjDhouUtNC3bCgYcTdiX',
url="http://localhost:7070",
threads=5,
qsize=500)
print engine_client.send_query({
'userId': 20,
"items": ['100', '101'],
"num": 10
})
#event_client.create_event(
#event="like",
#entity_type="user",
#entity_id='20',
#target_entity_type="item",
#target_entity_id='25'
#)
"""
Send sample query to prediction engine
"""
import predictionio
engine_client = predictionio.EngineClient(url="http://52.79.113.154:8000")
print "Sending query..."
# for i in range(1, 23):
# print "User: "******"user": "******" + str(i),
# "num": 20
# #"blackList" : ["i43"]
# }
# )
#
print engine_client.send_query({
"user": "******",
"limit": 30,
"skip": 0
#"blackList" : ["i43"]
})
# print engine_client.send_query(
# {
# "user": "******",
# "num": 10,
# "categories": ["c4", "c3"],
