def register():
form = Reg(request.form)
if request.method == 'GET': # make sure the method used is define above
return render_template('register.html', form=form), logging.warning(
"you are under the register page now using GET, well done bacco ")
elif request.method == 'POST' and form.validate():
name = form.name.data # the following are the data from the registration form
username = form.username.data
email = form.email.data
password = sha512_crypt.hash(str(
form.password.data)) # passsword is encrypted
# defining a new variable taking as input the values from the registration form
myuser = [{
"name": name,
"username": username,
"email": email,
"password": password
}]
# insert the list into the mongo db
x = mycol.insert_many(myuser), print("inserting this user: "******"in the database called ", mycol)
return render_template('register.html', form=form), print(
"you are under the register page now using POST, data are sent to database"
)
else:
print("this is the error")
def index():
form = Init(request.form)
if request.method == 'GET': # make sure the method used is define above
return render_template('home.html', form=form), logging.warning(
"you are under the home page now using GET, well done bacco ")
if request.method == 'POST' and form.validate():
# the following are the data from the init form
name = form.name.data
telefono = form.telefono.data
email = form.email.data
messaggio = form.messaggio.data
# defining a new variable taking as input the values from the init form
mymsg = [{
"name": name,
"telefono": telefono,
"email": email,
"messaggio": messaggio
}]
# insert the list into the mongo db
x = mycol.insert_many(mymsg), print("inserting this user: "******"in the database called ", mycol)
msg = Message(
'New message from: ',
sender='*****@*****.**',
recipients=['*****@*****.**'],
html=
f"<h3> new message from: </h3> <ul><li>NOME: {name}</li> <li>TELEFONO: {telefono}</li><li> EMAIL: {email}</li> <li> MESSAGGIO: {messaggio}</li> <li> DATA e ORA: {readtime}</li>"
)
mail.send(msg)
return render_template('home.html', form=form), print(
"you are under the home page now using POST, data are sent to database"
)
def signup():
form = Init(request.form)
if request.method == 'GET': # make sure the method used is define above
return render_template('signup.html', form=form), logging.warning(
"you are under the signup page now using GET, well done mrbacco ")
if request.method == "POST" and form.validate():
name = form.name.data
username = form.username.data
email = form.email.data
password = sha512_crypt.encrypt(str(form.password.data))
myuser = [{
"name": name,
"username": username,
"email": email,
"password": password, #this is the hashed password
"date": readtime,
}]
#checking if the username is already in use
#u = mycol_u.find_one({'username' : username})
u = cur.execute("SELECT * FROM users WHERE username='******'")
conn.commit()
if u is not None:
flash("USERNAME ALREADY IN USE!!, please choose another username!",
"danger")
return render_template('signup.html', form=form), print(
"reload the signup page due username already present")
else:
cur.execute(
"INSERT INTO users VALUES ('name', 'username', 'email', 'password')"
), print("inserting this item: ",
myuser) # insert user into the mongo db
# send an email to [email protected] for testing purposes: PLEASE DISABLE THIS IN PRODUCTION!!!!!
msg = Message(
"NEW MESSAGE: ",
sender='*****@*****.**',
recipients=["*****@*****.**"],
html=
f"<h3> new signup from: </h3> <ul> <li>name: {name}</li> <li>username: {username}</li> <li> email: {email}</li> <li> date and time: {readtime}</li>"
)
mail.send(msg)
flash("thanks for registering, you can now login", "success")
return redirect(
url_for('signin')), print("redirecting to signin page")
flash("Credential not correct, try again", "danger")
return render_template(
'signup.html',
form=form), print("reload the signup page due to failure")
print(" connected to email ... probably")
############## email server SETUP END ##############
############## db SETUP START ##############
# using mongo db cloud version
# checking the connection to cloud ongodb and printing in the console the list of collections under the database
try:
myclient = pymongo.MongoClient("mongodb://######:########@cluster0-shard-00-00-goutv.mongodb.net:27017,########utv.m#####)
mydb = myclient["lawyers"]
mycol = mydb["feedback"]
print("if connected to db, then these are the collections in mydb: ", mydb.list_collection_names()) #used to check if db is connected
except:
logging.warning("Could not connect to MongoDB")
############## db SETUP END ##############
############## defining the routes for the different web pages START ##############
class Init(Form): #definition of a class for the init form
name = StringField('Name', [validators.Length(min = 1, max = 100)])
telefono = StringField('Telefono', [validators.Length(min = 5, max = 50)])
email = StringField('Email', [validators.Length(min = 6, max = 50)])
messaggio = StringField('Messaggio', [validators.DataRequired()])
@app.route("/", methods = ['GET', 'POST'])
def index():
form = Init(request.form)
if request.method == 'GET': # make sure the method used is define above
return render_template('home.html', form = form), logging.warning("you are under the home page now using GET, well done bacco ")
) # creating an instalnce of the Flask class for thsi app as web server
############## db SETUP START ##############
# using mongo db cloud version
# checking the connection to cloud ongodb and printing in the console the list of collections under the database
try:
myclient = pymongo.MongoClient(
"mongodb://*****:*****@cluster0-shard-00-00-goutv.mongodb.net:27017,cluster0-shard-00-01-goutv.mongodb.net:27017,cluster0-shard-00-02-goutv.mongodb.net:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin&retryWrites=true&w=majority"
)
mydb = myclient["database001"]
mycol = mydb["python001"]
print("if connected to db, then these are the collections in mydb: ",
mydb.list_collection_names()) #used to check if db is connected
except:
logging.warning("Could not connect to MongoDB")
############## db SETUP END ##############
############## function to protect views if not logged in START ##############
def isAuth(func):
@wraps(func)
def wrap(*args, **kwargs):
if "logged_in" in session:
return func(*args, **kwargs)
else:
error = 'Need to login, first'
return redirect(url_for("login"), error), print(
"user not registered, redirecting now to login page")
def dashboard():
form = Scrape(request.form)
if request.method == 'GET': # make sure the method used is define above
return render_template('dashboard.html', form=form), logging.warning(
"you are under the home page now using GET, well done mrbacco ")
if request.method == 'POST' and form.validate():
# the following are the data from the init form
url = form.url.data
u_name = session['username']
result = requests.get(url) # getting the url from the webform
print("the requested url is: ",
url) # printing the url to make sure the variable contains it
print("the response code is: ", result.status_code)
print(result.headers)
# now I can apply the BS4 class to the content of the page
payload = result.content # defining a new variable that takes the content of the web page
soup = BeautifulSoup(
payload, "lxml"
) # created "soup": the beautiful soup object to use for scraping
links = [
] # I'm creating an empty list that will be filled with the result of the findings
# this is the actual block of code for the core web scraping
'''
for var in soup.find_all("div"): # looping to find all the "div" of the page
a_tag = var.find_all("a")
links.append(a_tag)
'''
for www in soup.find_all(
'a'
): # looking for all the hyperlinks in the page and printing them
print("the following are the hypelinks available: ",
www.get('href'))
for para in soup.find_all(
'p'
): # looping to find all the "paragraphs" of the page and printing the results
print("the paragraphs are: ", str(para.text))
# print(links,"\n")
# value = [a.text for a in soup.find_all("links")] #looping to find all the links in the page references
# values.append(value)
# tot = values.count("...")
# pprint.pprint(values) #using .text allows to extract only the text on the webpage and not the tags
print("the requested title name is :", soup.title.name)
print("the requested title is :", soup.title)
print("the requested title parent name is :", soup.title.parent.name)
# defining a new variable taking as input the values from the init form to populate the DB
mymsg = [{
"url": url,
"response code": result.status_code,
"username": u_name,
"date": readtime,
}]
x = mycol.insert_many(mymsg), print(
"inserting this item: ",
mymsg) # insert the list into the mongo db
"""
# send an email to [email protected] for testing purposes: PLEASE DISABLE THIS IN PRODUCTION!!!!!
msg = Message("NEW MESSAGE: ", sender='*****@*****.**', recipients=["*****@*****.**"], html = f"<h3> new message from: </h3> <ul><li>URL: {url}</li> <li> EMAIL: {email}</li> <li> DATA e ORA: {readtime}</li>")
mail.send(msg)
validating the url in the form http://...
url1="http://"+url #making sure the parser will get the HTTP://WWW.EXAMPLE.COM syntax
"""
print("you are under the dashboard page now")
return render_template('dashboard.html', form=form)
############## db SETUP START ##############
# using mongo db cloud version
# checking the connection to cloud ongodb and printing in the console the list of collections under the database
try:
myclient = pymongo.MongoClient(
"mongodb://*****:*****@cluster0-shard-00-00-goutv.mongodb.net:27017/test?ssl=true&replicaSet=Cluster0-shard-0&authSource=admin&retryWrites=true&w=majority"
)
mydb = myclient["webscraping"]
mycol = mydb["scraping"]
mycol_u = mydb["users"]
print("if connected to db, then these are the collections in mydb: ",
mydb.list_collection_names()) # used to check if db is connected
except:
logging.warning("not connected to mongodb")
# using SQLite3 for heroku deployment
conn = sqlite3.connect("webscraping.db")
cur = conn.cursor()
'''
cur.execute(""" CREATE TABLE scraping1 (
url text,
response_code test,
username text
)""")
cur.execute(""" CREATE TABLE users (
name text,
username test,