def test_site(self):
delegate = XDelegate()
# session = delegate.get_session()
# Site 1
site1 = Site()
site1.name = "Site1"
site1.url = 'http://foo.com'
delegate.site_create(site1)
siteId = site1.id
assert siteId > 0
# Site 2
site2 = Site()
site2.name = "Site2"
site2.url = 'http://foo.com'
delegate.site_create(site2)
# # User.query.all()
# f = session.query(Site).first()
# assert f.id > 0
sites = delegate.site_get_all()
print("No of site: {}".format(len(sites)))
assert len(sites) == 2
delegate.site_delete_all()
def login():
log.info("Logging in")
delegate = Delegate()
error = None
if request.method == 'POST':
email = request.form['email']
password = request.form['password']
user = delegate.user_get_by_email_and_password(email, password)
if user == None or False:
error = 'Invalid Credentials. Please try again.'
else:
session['logged_in'] = True
session['user_id'] = user.id
# Get first selected site
if user.current_site_id == None:
sites = delegate.site_get_all()
if len(sites) > 0:
current_site = sites[0]
user.current_site_id = current_site.id
delegate.user_update(user)
flash('You were just logged in!')
return redirect(url_for('home'))
return render_template('login.html', error=error)
def personal_settings_edit():
delegate = Delegate()
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
return render_template('personal_settings_edit.html',
user=user,
sites=sites,
clazz=User)
def status():
delegate = Delegate()
status = []
#Home folder
from os.path import expanduser
home = expanduser("~")
status.append({'key': 'Home folder', 'value': home})
#Python version
import sys
status.append({'key': 'Python version', 'value': sys.version})
#Current folder
status.append({
'key': 'Current folder',
'value': os.path.dirname(__file__)
})
#Graphviz dependency
from shutil import which
status.append({
'key': 'Graphviz present',
'value': which('dot') is not None
})
# SQLAlchemy version
status.append({
'key': 'SQLAlchemy version',
'value': sqlalchemy.__version__
})
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
db_status = {}
db_status['size'] = 'N/A' if isinstance(
database.engine.pool,
sqlalchemy.pool.impl.NullPool) else database.engine.pool.size()
db_status['checkedin'] = 'N/A' if isinstance(
database.engine.pool,
sqlalchemy.pool.impl.NullPool) else database.engine.pool.checkedin()
db_status['overflow'] = 'N/A' if isinstance(
database.engine.pool,
sqlalchemy.pool.impl.NullPool) else database.engine.pool.overflow()
db_status['checkedout'] = 'N/A' if isinstance(
database.engine.pool,
sqlalchemy.pool.impl.NullPool) else database.engine.pool.checkedout()
return render_template('status.html',
status=status,
user=user,
sites=sites,
db_status=db_status)
def users():
delegate = Delegate()
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
users = delegate.user_get_all()
return render_template('users.html',
user=user,
sites=sites,
users=users,
clazz=User)
def crawl():
delegate = Delegate()
user = delegate.user_get_by_id(session['user_id'])
site = delegate.site_get_by_id(user.current_site_id)
crawls = delegate.crawl_get_all_for_site(user.current_site_id)
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
return render_template('crawl.html',
crawls=crawls,
site=site,
user=user,
sites=sites)
def crawl_exe():
global PROGRESS_TRACKER
global CRAWLS
delegate = Delegate()
@copy_current_request_context
def notify(msg):
crawlId = str(msg['crawlId'])
# FIXME: Naive way to avoid leaks (byt keeping references)
if msg['status'] == 'done':
del CRAWLS[int(crawlId)]
# progress = ProgressTracker._msg_to_progress(msg)
# pj = jsonpickle.encode(progress)
PROGRESS_TRACKER.set_progress(crawlId, msg)
try:
crawl = delegate.crawl_get_by_id(crawlId)
except ValueError as ve:
flash('No crawl id.')
return redirect(url_for('crawl'))
if not request.form['address']:
flash('No address.')
return redirect(url_for('crawl'))
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
# Save to DB
crawl = Crawl(site_id=user.current_site_id)
delegate.crawl_create(crawl)
initial_url = request.form['address']
max_links = 0
if len(request.form['max']) > 0 and int(request.form['max']) > 0:
max_links = int(request.form['max'])
crawler = CrawlerDB(delegate,
initial_url,
id=crawl.id,
no_workers=10,
max_links=max_links)
CRAWLS[crawl.id] = crawler
crawler.addListener(notify)
crawler.start()
return render_template('crawl_progress.html',
crawl=crawl,
user=user,
sites=sites)
def site_delete():
delegate = Delegate()
site_id = request.args.get('site_id', type=int)
# page = request.args.get('page', type=str)
delegate.site_delete_by_id(site_id)
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
flash('Site deleted !')
return render_template('sites.html', user=user, sites=sites)
def user_edit():
delegate = Delegate()
user_id = request.args.get('user_id', type=int)
edited_user = delegate.user_get_by_id(user_id)
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
users = delegate.user_get_all()
return render_template('user_edit.html',
user=user,
sites=sites,
users=users,
clazz=User,
edited_user=edited_user)
def internal_linking():
delegate = Delegate()
user = delegate.user_get_by_id(session['user_id'])
# site = delegate.site_delete_by_id(user.current_site_id)
last_crawl = delegate.crawl_get_last_for_site(user.current_site_id)
last_crawl_id = 'null' # Values for JavaScript
if last_crawl is not None:
last_crawl_id = last_crawl.id
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
return render_template('internal_linking.html',
last_crawl_id=last_crawl_id,
user=user,
sites=sites)
def crawl_view_links():
delegate = Delegate()
try:
id = request.args.get('id', type=int)
crawl = delegate.crawl_get_by_id(id)
links = delegate.url_get_all_by_crawl_id(id)
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
return render_template('crawl_view_links.html',
crawl=crawl,
links=links,
user=user,
sites=sites)
except ValueError as ve:
flash('No crawl id.')
return redirect(url_for('crawl'))
def test_crawl(self):
delegate = XDelegate()
print("test_crawl started")
# session = delegate.get_session()
# Site 1
site1 = Site()
site1.name = "Site1"
site1.url = 'http://foo.com'
delegate.site_create(site1)
# Crawl
crawl = Crawl(site_id=site1.id)
delegate.crawl_create(crawl)
assert crawl.id > 0
# Create a datetime 2 minutes in the past
delta = datetime.timedelta(minutes=-2)
t2 = crawl.date - delta
crawl2 = Crawl(site_id=site1.id, date=t2)
delegate.crawl_create(crawl2)
assert crawl2.id > 0
sites = delegate.site_get_all()
print("No of site: {}".format(len(sites)))
assert len(sites) == 1
crawls = delegate.crawl_get_all()
assert len(crawls) == 2
crawls2 = delegate.crawl_get_all_for_site(site1.id)
assert len(crawls) == 2
last_crawl = delegate.crawl_get_last_for_site(site1.id)
assert last_crawl.id == crawl2.id, "Last crawl id was {} when it should be {}".format(
last_crawl.id, crawl2.id)
# delegate.crawl_delete_all()
delegate.site_delete_all()
print("test_crawl done")
def home():
delegate = Delegate()
user = delegate.user_get_by_id(session['user_id'])
sites = delegate.site_get_all(
) # TODO: In the future show only sites for current user
return render_template('home.html', sites=sites, user=user)