def setUp(self):
"""初始化数据库表结构， 完成数据库session，缓存及redis连接"""
Dao.init_db_uri()
self.session = Dao.session()
Dao.init_table_schema()
self.cache = OPCache()
self.rds = Dao.redis()
self.session.rollback()
login_url='/login/',
)
tornado.web.Application.__init__(self, urls, **settings)
if __name__ == '__main__':
""""""
options.define(name='config', default='dev')
options.define(name='port', default=38001)
options.define(name='process', default=1)
options.define(name='server', default='total')
tornado.options.parse_command_line()
config = Config(options.config)
Dao.init_db_uri(options.config)
# TODO for dev and test
Dao.init_schema()
app = Application()
app.config = options.config
log.info('Starting box api server... Listening port: {}'.format(options.port))
try:
server = HTTPServer(app, xheaders=True)
server.bind(int(options.port))
server.start(num_processes=int(options.process))
tornado.ioloop.IOLoop.instance().start()
except Exception as e:
log.error('box api can not running:\n{}'.format(e), exc_info=True)
tornado.ioloop.IOLoop.instance().stop()
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
# from sqlalchemy import create_engine
#
# DB_URL = 'mysql://*****:*****@127.0.0.1/yiqidingtest?charset=utf8'
#
# conn = create_engine(DB_URL, encoding='utf8')
from models.fun_video import FunVideo
from models.dao import Dao
# from utils.utils import get_funtv_mp4
from utils.pinyin import pinyin
Dao.init_db_uri('rory')
db_session = Dao.db_session()
class FunMovieSpider(CrawlSpider):
"""
风行视频爬取
"""
name = 'fun_movie'
allows_domains = [
'fun.tv',
]
start_urls = [
'http://www.fun.tv/retrieve/c-e794b5e5bdb1.n-e5bdb1e78987.o-pl.pg-1'
]
from scrapy.selector import Selector
try:
from scrapy.spider import Spider
except ImportError, e:
from scrapy.spider import BaseSpider as Spider
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
from models.fun_video import FunVideo, SubFunViedo
from models.dao import Dao
from utils.pinyin import pinyin
Dao.__echo = False
Dao.init_db_uri('rory')
Dao.init_schema()
db_session = Dao.db_session()
class FunTVSpider(CrawlSpider):
"""
风行电视爬取
"""
name = 'fun_cartoon'
allows_domains = ['fun.tv', ]
start_urls = [
'http://www.fun.tv/retrieve/c-e58aa8e6bcab.n-e5bdb1e78987.o-pl.pg-1'
]
)
# self.db_session = scoped_session(sessionmaker(bind=Dao.engine()))
tornado.web.Application.__init__(self, urls, **settings)
if __name__ == '__main__':
""""""
options.define(name='config', default='TT')
options.define(name='port', default=8002)
options.define(name='process', default=1)
options.define(name='server', default='total')
tornado.options.parse_command_line()
config = Config(options.config)
Dao.init_db_uri(options.config)
# for dev and test
# if options.config != __release__:
# Dao.init_schema()
app = Application()
app.config = options.config
app.rds = Dao.rds()
log.info('Starting {} server... Listening port: {}'.format(
options.server, options.port))
try:
server = HTTPServer(app, xheaders=True)
server.bind(int(options.port))
