# coding: utf-8
from utils import util
from cdfbj.subscriber import Subscriber
if __name__ == '__main__':
util.config_logger('cdfbj_subscriber')
import config
reminder = Subscriber(config.CDFBJ_SUBSCRIBER_CONFIG)
reminder.execute()
pass
def _init_logger(self):
util.config_logger()
self.logger = logging.getLogger(__name__)
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
logger = logging.getLogger(__name__)
engine = create_engine(config.MYSQL_CONFIG_TESTING['DB_CONNECT_STRING'], echo=False)
session_cls = sessionmaker(bind=engine)
Base = declarative_base()
def init_db():
from sql.base import CdfBjUserInfo
Base.metadata.create_all(bind=engine)
pass
def init_db_command():
"""Clear existing data and create new tables."""
init_db()
logger.info("Initialized the database.")
if __name__ == "__main__":
util.config_logger('db')
init_db()
pass
# coding: utf-8
from utils import util
from homelink.spiders.sx import SxSpider
from homelink.spiders.hz import HzSpider
from homelink.spiders.nj import NjSpider
from homelink.spiders.nj2 import Nj2Spider
from homelink.spiders.hz2 import Hz2Spider
from homelink.spiders.community import CommunitySpider
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
if __name__ == '__main__':
util.config_logger()
process = CrawlerProcess(get_project_settings())
process.crawl(SxSpider)
process.crawl(HzSpider)
process.crawl(NjSpider)
process.crawl(Nj2Spider)
process.crawl(Hz2Spider)
process.crawl(CommunitySpider)
process.start()
pass
# 更新下次统计的日期
self._next_report_time = self._next_report_time + datetime.timedelta(
days=1)
except Exception as e:
logger.info('report exception[{0}].'.format(e))
def execute(self):
# 初始化
self.__init()
while True:
try:
time.sleep(10)
self.sql_helper.begin_session()
self.__push_ips()
self.__update()
self.__report()
self.sql_helper.close_session()
except Exception as e:
logger.exception('execute [{0}] exception[{1}].'.format(
self.name, e))
if __name__ == "__main__":
util.config_logger('ip manager')
import config
manager = IpManager(config.IP_MANAGER_CONFIG)
manager.execute()
pass