# coding: utf-8 from utils import util from cdfbj.subscriber import Subscriber if __name__ == '__main__': util.config_logger('cdfbj_subscriber') import config reminder = Subscriber(config.CDFBJ_SUBSCRIBER_CONFIG) reminder.execute() pass
def _init_logger(self): util.config_logger() self.logger = logging.getLogger(__name__)
from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base logger = logging.getLogger(__name__) engine = create_engine(config.MYSQL_CONFIG_TESTING['DB_CONNECT_STRING'], echo=False) session_cls = sessionmaker(bind=engine) Base = declarative_base() def init_db(): from sql.base import CdfBjUserInfo Base.metadata.create_all(bind=engine) pass def init_db_command(): """Clear existing data and create new tables.""" init_db() logger.info("Initialized the database.") if __name__ == "__main__": util.config_logger('db') init_db() pass
# coding: utf-8 from utils import util from homelink.spiders.sx import SxSpider from homelink.spiders.hz import HzSpider from homelink.spiders.nj import NjSpider from homelink.spiders.nj2 import Nj2Spider from homelink.spiders.hz2 import Hz2Spider from homelink.spiders.community import CommunitySpider from scrapy.crawler import CrawlerProcess from scrapy.utils.project import get_project_settings if __name__ == '__main__': util.config_logger() process = CrawlerProcess(get_project_settings()) process.crawl(SxSpider) process.crawl(HzSpider) process.crawl(NjSpider) process.crawl(Nj2Spider) process.crawl(Hz2Spider) process.crawl(CommunitySpider) process.start() pass
# 更新下次统计的日期 self._next_report_time = self._next_report_time + datetime.timedelta( days=1) except Exception as e: logger.info('report exception[{0}].'.format(e)) def execute(self): # 初始化 self.__init() while True: try: time.sleep(10) self.sql_helper.begin_session() self.__push_ips() self.__update() self.__report() self.sql_helper.close_session() except Exception as e: logger.exception('execute [{0}] exception[{1}].'.format( self.name, e)) if __name__ == "__main__": util.config_logger('ip manager') import config manager = IpManager(config.IP_MANAGER_CONFIG) manager.execute() pass