def setUp(self): """初始化数据库表结构, 完成数据库session,缓存及redis连接""" Dao.init_db_uri() self.session = Dao.session() Dao.init_table_schema() self.cache = OPCache() self.rds = Dao.redis() self.session.rollback()
login_url='/login/', ) tornado.web.Application.__init__(self, urls, **settings) if __name__ == '__main__': """""" options.define(name='config', default='dev') options.define(name='port', default=38001) options.define(name='process', default=1) options.define(name='server', default='total') tornado.options.parse_command_line() config = Config(options.config) Dao.init_db_uri(options.config) # TODO for dev and test Dao.init_schema() app = Application() app.config = options.config log.info('Starting box api server... Listening port: {}'.format(options.port)) try: server = HTTPServer(app, xheaders=True) server.bind(int(options.port)) server.start(num_processes=int(options.process)) tornado.ioloop.IOLoop.instance().start() except Exception as e: log.error('box api can not running:\n{}'.format(e), exc_info=True) tornado.ioloop.IOLoop.instance().stop()
from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle # from sqlalchemy import create_engine # # DB_URL = 'mysql://*****:*****@127.0.0.1/yiqidingtest?charset=utf8' # # conn = create_engine(DB_URL, encoding='utf8') from models.fun_video import FunVideo from models.dao import Dao # from utils.utils import get_funtv_mp4 from utils.pinyin import pinyin Dao.init_db_uri('rory') db_session = Dao.db_session() class FunMovieSpider(CrawlSpider): """ 风行视频爬取 """ name = 'fun_movie' allows_domains = [ 'fun.tv', ] start_urls = [ 'http://www.fun.tv/retrieve/c-e794b5e5bdb1.n-e5bdb1e78987.o-pl.pg-1' ]
from scrapy.selector import Selector try: from scrapy.spider import Spider except ImportError, e: from scrapy.spider import BaseSpider as Spider from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle from models.fun_video import FunVideo, SubFunViedo from models.dao import Dao from utils.pinyin import pinyin Dao.__echo = False Dao.init_db_uri('rory') Dao.init_schema() db_session = Dao.db_session() class FunTVSpider(CrawlSpider): """ 风行电视爬取 """ name = 'fun_cartoon' allows_domains = ['fun.tv', ] start_urls = [ 'http://www.fun.tv/retrieve/c-e58aa8e6bcab.n-e5bdb1e78987.o-pl.pg-1' ]
) # self.db_session = scoped_session(sessionmaker(bind=Dao.engine())) tornado.web.Application.__init__(self, urls, **settings) if __name__ == '__main__': """""" options.define(name='config', default='TT') options.define(name='port', default=8002) options.define(name='process', default=1) options.define(name='server', default='total') tornado.options.parse_command_line() config = Config(options.config) Dao.init_db_uri(options.config) # for dev and test # if options.config != __release__: # Dao.init_schema() app = Application() app.config = options.config app.rds = Dao.rds() log.info('Starting {} server... Listening port: {}'.format( options.server, options.port)) try: server = HTTPServer(app, xheaders=True) server.bind(int(options.port))