def start_analysis():
error=None
if request.method == 'POST':
if request.form.get('transactionFile'):
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the apriori algorithm will be called.')
if (session['userSector'] == "Bakery"):
filename="goods.csv"
if (session['userSector'] == "Pharmacy"):
filename="pharmacy.csv"
apriori.mainAlgorithm(filename)
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the apriori algorithm ends.')
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the advices generation will be called.')
advice.generate_advice()
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the advices generation ends.')
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the advices generation will be getted.')
collectionName = 'Customer_'+session['userId']
customerDB = connection.get_collection(collectionName)
advicesDB=customerDB['Advices']
advices = advicesDB.distinct('sentence')
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the advices will be displayed.')
if (session['userSector'] == "Bakery"):
return render_template('analysis.html', advices=advices)
if (session['userSector'] == "Pharmacy"):
return render_template('analysis2.html', advices=advices)
else:
error = 'No file selected'
return render_template('startAnalysis.html', error=error)
#transaction = request.get('transactionFile'allow_multiple=True)
#for t in transaction:
#return redirect(url_for('analysis'))
else:
return render_template('startAnalysis.html')
#
# analysisFile = 'apriori_data.csv'
# flash('Successfully %s' % analysisFile)
# return redirect(url_for('analysis'))
# else:
# flash('Failed to analyse %s' % analysisFile)
# return redirect(url_for('startAnalysis'))
def upload_file():
if request.method == "POST":
file = request.files["file"]
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the check file has to start.')
if file and allowed_file(file.filename):
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the check file is done.")
filename = secure_filename(file.filename)
file.save(os.path.join(app.config["UPLOAD_FOLDER"], filename))
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is the cleaning of data has to start.")
validate.validation(filename)
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the cleaning of data is done.")
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the parsing process has to start (transform to JSON).")
json_filename = parse.parse_file(filename)
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the parsing process (transform to JSON) is done.")
print(session["userId"])
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the data storage process has to start.")
collectionName = "Customer_" + session["userId"]
customerDB = connection.get_collection(collectionName)
transactionsDB = customerDB["Transactions"]
print(transactionsDB)
input_file = open(json_filename)
for line in input_file:
transactionsDB.insert(json.loads(line))
flash("Successfully uploaded %s" % filename)
logging.basicConfig(format="%(asctime)s %(message)s")
logging.warning("is when the data storage process is done.")
return redirect(url_for("upload_file"))
else:
flash("Invalid file %s" % file.filename)
return redirect(url_for("upload_file"))
else:
files = notify.get_files(app.config["UPLOAD_FOLDER"])
return render_template("upload.html", files=files)
return redirect(url_for("upload_file"))
def generate_advice():
templateDB= db['advice_templates']
#collectionName = 'Customer_568e84f322c8e50c448e07a7'
collectionName = 'Customer_'+session['userId']
#print (collectionName)
#download the customer collection
customerDB = connection.get_collection(collectionName)
#print (customerDB)
#access to the customer rules collection where the rules will be stored
rulesDB=customerDB['Rules']
advicesDB=customerDB['Advices']
#print(advicesDB)
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the analyse of the pattern will be executed.')
rulesLocation = rulesDB.find({'confidence':{ "$gt": 0.95 }})
#rulesLocation = rulesDB.find({'confidence': 1})
rulesPromotional = rulesDB.find({'confidence':{ "$lt": 0.95 }})
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the analyse of the pattern is done.')
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the extraction of the templates will be executed.')
adviceLocation = advice_templates.find({'name':'placement'})
for a in adviceLocation:
adviceTxt1= a.get("txt")
advicePromotional = advice_templates.find({'name':'promotional'})
for a in advicePromotional:
adviceTxt2= a.get("txt")
for rule in rulesLocation:
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the generation of the advice will start.')
productTxt = 'The products '+str(rule.get("explained product"))+ ' and '+ str(rule.get("explaining product"))+' are frequently bought together.'
#resAdvice = create_placement_advice(productTxt)
resAdvice = productTxt+ '\n'+ adviceTxt1+ '\n\n\n'
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the advice is generated.')
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the advice will be stored.')
newAdvice = {'sentence': resAdvice, 'type': 'placement'}
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the advice is stored.')
advicesDB.insert(newAdvice)
#print(resAdvice)
for rule in rulesPromotional:
productTxt = 'The products '+str(rule.get("explained product"))+ ' and '+ str(rule.get("explaining product"))+' are frequently bought together.'
resAdvice = productTxt+ '\n'+ adviceTxt2+ '\n\n\n'
# resAdvice = create_promotional_advice(productTxt)
newAdvice = {'sentence': resAdvice, 'type': 'promotional'}
advicesDB.insert(newAdvice)
print(resAdvice)
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the advices have been generated and stored.')
def mainAlgorithm(inputFilename):
######## Connection to the customer collection
# get the name of the customer collection using the session
#collectionName = 'Customer_568d3b7222c8e507363c1f6f'
collectionName = 'Customer_'+session['userId']
#print (collectionName)
#download the customer collection
customerDB = connection.get_collection(collectionName)
#print (customerDB)
# INIT PARAMETERS before running the Apriori Algorithm
#prepare the transactional data in a csv file (apriori_data.csv)
# logging.basicConfig(format='%(asctime)s %(message)s')
# logging.warning('is when the input data transformation is called.')
transform.prepare_data(customerDB)
filename = 'apriori_data.csv'
#access to the customer rules collection where the rules will be stored
rulesDB=customerDB['Rules']
#print(rulesDB)
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the running of the data mining Apriori Algorithm is starting.')
goods = defaultdict(list)
num_args = 3
minSup = minConf = 0
noRules = False
# Make sure the right number of input files are specified
#print sys.path
dataset = csv.reader(open(filename, "r"))
goodsData = csv.reader(open(inputFilename, "r"))
# minSup = aMinSup
minSup = 0.2
# minConf = aMinConf
minConf = 0.7
csvfile = open('rules.csv', 'wb')
writer = csv.writer(csvfile, delimiter=' ')
print "Dataset: ", filename, " MinSup: ", minSup, " MinConf: ", minConf
row= "Dataset: "+ filename+ " MinSup: "+ str(minSup)+ " MinConf: "+ str(minConf)
writer.writerow(row)
print "=================================================================="
row= "=================================================================="
writer.writerow(row)
for item in goodsData:
goods[item[0]] = item[1:]
a = Apriori(dataset, minSup, minConf)
frequentItemsets = a.genAssociations()
count = 0
for k, item in frequentItemsets.iteritems():
for i in item:
if k >= 2:
count += 1
print(str(count)+": "+readable(i, goods)+"\tsupport=",a.support(a.freqList[i]))
print "Skyline Itemsets: ", count
if not noRules:
rules = a.genRules(frequentItemsets)
for i, rule in enumerate(rules):
print "Rule",i+1,":\t ",readable(rule[0], goods),"\t-->",readable(rule[1], goods),"\t [sup=",rule[2]," conf=",rule[3],"]"
row= "Rule",i+1,":\t ",readable(rule[0], goods),"\t-->",readable(rule[1], goods),"\t [sup=",rule[2]," conf=",rule[3],"]"
writer.writerow(row)
newRule = {"rule number": i+1, "explained product":readable(rule[0], goods), "explaining product": readable(rule[1], goods), "support": rule[2], "confidence":rule[3]}
ruleid = rulesDB.insert(newRule)
print (str(ruleid))
print "\n"
csvfile.close()
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when the running of the data mining Apriori Algorithm ends.')