def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
#Set database up
engine = create_engine(
'sqlite:///../database/hegemone_dev.sqlite3',
echo=False,
#Avoid exception caused by sqlites poor support of multithreading.
#This needs to be removed if the system is used by many users and when
#another database is used.
connect_args={'check_same_thread': False},
poolclass=StaticPool)
db.setup_db(engine)
Session = sessionmaker(bind=engine)
session = Session()
#Set repositories up
website_repository = WebsiteRepository(session)
word_count_repository = WordCountRepository(session)
#Set business logic layer up
businessLogicLayer.websites.website_repository = website_repository
businessLogicLayer.wordCounts.word_count_repository = word_count_repository
#Set webserver up
config = Configurator(settings=settings)
config.include('pyramid_jinja2')
config.add_static_view('static', 'static', cache_max_age=3600)
routes.set_routes(config)
config.scan()
return config.make_wsgi_app()
def setUp(self):
engine = create_engine('sqlite:///:memory:', echo=False)
db.setup_db(engine)
Session = sessionmaker(bind=engine)
self.session = Session()
self.category_repository = CategoryRepository(self.session)
self.website_repository = WebsiteRepository(self.session)
from dataAccessLayer.repositories.websiteRepository import WebsiteRepository
from dataAccessLayer.repositories.wordCountRepository import WordCountRepository
from websiteToText import converter
from textMetrices import WordFrequency
from dataAccessLayer.helper import db
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
if __name__ == "__main__":
#Set database up
engine = create_engine('sqlite:///../../database/hegemone_dev.sqlite3',
echo=False)
db.setup_db(engine)
Session = sessionmaker(bind=engine)
session = Session()
#Set repositories up
website_repository = WebsiteRepository(session)
word_count_repository = WordCountRepository(session)
#Get websites
websites = website_repository.get_all_websites()
#Crawl websites
for website_tuple in websites:
raw_text = converter.get_text(website_tuple[1], 0)
word_counts = WordFrequency.count_words(raw_text)
#Store word counts in database
website_id = website_tuple[0]
