def run(): logger = logging.getLogger('main') logger.info('start tori.fi monitoring') consumers = [{ 'consumer': ToriConsumer(CarParser(), NUM_PAGES_TO_SEARCH) }, { 'consumer': ToriConsumer(ToriParser(), NUM_PAGES_TO_SEARCH) }] for c in consumers: c['old'] = c['consumer'].parser.list_factory('old', db=DBFactory.create(), populate=True) logger.info('startup topic={}, {}'.format( c['consumer'].parser.topic, _list_to_daterangetext(c['old']))) while True: for c in consumers: new_items = get_new_items(c) new_items.check_for_alarms() new_items.persist() if len(new_items): logger.info(f'\n{new_items}') logger.info('topic={}, {}/{} items ({} added), {}'.format( c['consumer'].parser.topic, len(c['old']), NUM_KEEP_ITEMS, len(new_items), _list_to_daterangetext(c['old']))) wait_time = 180 + random.randint(1, 60) time.sleep(wait_time)
def connection(database=None, transaction=None, autoconnect=True, doraise=True): conn = DBFactory().connection(database, transaction) if not conn and doraise: raise ErrorDatabase('Can not get active connection by name "%s"' % (database)) return conn
def get_new_items(fetcher): new_items = fetcher['consumer'].parser.list_factory('new', db=DBFactory.create()) for items in fetcher['consumer']: diff_items = fetcher['old'].diff_to(items) if len(diff_items) == 0: break fetcher['old'] += diff_items new_items += diff_items new_items.sort_by_date() fetcher['consumer'].enrich(new_items) fetcher['old'].truncate_oldest(NUM_KEEP_ITEMS) return new_items
def set_debuglevel(level): DBFactory().set_debuglevel(level)
def initialize(): DBFactory().initialize()
def SetConfig(config): """ config: dict as MySQLdb.connect parameters """ return DBFactory().set_config(config)
def close(): """ 清除当前线程缓冲连接 """ DBFactory().cache.clear()
def reset(): """ 清除当前线程缓冲连接 """ DBFactory().cache.clear()
def rollback(database=None, transaction=None, doraise=False): db = DBFactory() for conn in db.cache.connections: conn.rollback() db.pushback(conn)
def commit(database=None, transaction=None): db = DBFactory() for conn in db.cache.connections: conn.commit() db.pushback(conn)