def setup(cls):
db_dir = os.path.dirname(cls.db_location)
if not os.path.exists(db_dir):
Path(db_dir).mkdir(parents=True, exist_ok=True)
db.init(cls.db_location)
db.connect()
db.create_tables(tables)
def execute_before_request():
"""
Thực thi trước khi xử lý request. Ví dụ: kết nối database.
:return:
"""
db.create_tables([
ProjectModel, ReportWeeklyModel, TeacherModel, StudentModel,
ReportModel, CommentModel, UserModel
],
safe=True)
from scrapy.crawler import CrawlerProcess
from src.models import db
from src.models import Url
from src.crawler import MySpider
if __name__ == "__main__":
# Create table if doesn't exist yet
db.create_tables([Url])
# Begin crawl process
process = CrawlerProcess({
'USER_AGENT':
"""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139
Safari/537.36"""
})
process.crawl(MySpider)
process.start()
def __init__(self, db_name):
self.logger = get_logger("db")
db.init(db_name)
db.connect()
db.create_tables([ProfileModel, WaypointModel, SequenceModel])
self.logger.debug("Connected to database")
root.setLevel(logging.INFO)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
ch.setFormatter(
ColoredFormatter(
'[%(log_color)s%(levelname)s%(reset)s] %(bold)s%(site_name)s%(reset)s'
' (%(videos_current_number)s): %(message)s'))
root.addHandler(ch)
return root
if __name__ == '__main__':
db.connect()
db.create_tables([Site, Video, Tag, VideoTag], safe=True)
logger = create_logger()
loop = asyncio.get_event_loop()
crawlers = [
YoujizzCrawler(),
]
try:
tasks = [crawler.crawl() for crawler in crawlers]
loop.run_until_complete(asyncio.wait(tasks))
except KeyboardInterrupt:
loop.close()
def create_db():
db.create_tables([
ProjectModel, ReportWeeklyModel, TeacherModel, StudentModel,
ReportModel, CommentModel, UserModel
],
safe=True)