def setup(cls): db_dir = os.path.dirname(cls.db_location) if not os.path.exists(db_dir): Path(db_dir).mkdir(parents=True, exist_ok=True) db.init(cls.db_location) db.connect() db.create_tables(tables)
def execute_before_request(): """ Thực thi trước khi xử lý request. Ví dụ: kết nối database. :return: """ db.create_tables([ ProjectModel, ReportWeeklyModel, TeacherModel, StudentModel, ReportModel, CommentModel, UserModel ], safe=True)
from scrapy.crawler import CrawlerProcess from src.models import db from src.models import Url from src.crawler import MySpider if __name__ == "__main__": # Create table if doesn't exist yet db.create_tables([Url]) # Begin crawl process process = CrawlerProcess({ 'USER_AGENT': """Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36""" }) process.crawl(MySpider) process.start()
def __init__(self, db_name): self.logger = get_logger("db") db.init(db_name) db.connect() db.create_tables([ProfileModel, WaypointModel, SequenceModel]) self.logger.debug("Connected to database")
root.setLevel(logging.INFO) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) ch.setFormatter( ColoredFormatter( '[%(log_color)s%(levelname)s%(reset)s] %(bold)s%(site_name)s%(reset)s' ' (%(videos_current_number)s): %(message)s')) root.addHandler(ch) return root if __name__ == '__main__': db.connect() db.create_tables([Site, Video, Tag, VideoTag], safe=True) logger = create_logger() loop = asyncio.get_event_loop() crawlers = [ YoujizzCrawler(), ] try: tasks = [crawler.crawl() for crawler in crawlers] loop.run_until_complete(asyncio.wait(tasks)) except KeyboardInterrupt: loop.close()
def create_db(): db.create_tables([ ProjectModel, ReportWeeklyModel, TeacherModel, StudentModel, ReportModel, CommentModel, UserModel ], safe=True)