def cleaner(self): private_nets = [ '0.%', '127.%', '192.168.%', '10.%', '172.16.%', '172.17.%', '172.18.%', '172.19.%', '172.20.%', '172.21.%', '172.22.%', '172.23.%', '172.24.%', '172.25.%', '172.26.%', '172.27.%', '172.28.%', '172.29.%', '172.30.%', '172.31.%' ] logger.info('Resolver cleaner run') history_del = History.select(History.id).order_by( History.id.desc()).offset(self.cfg.DiffCount()) count = DNSResolver.delete().where( DNSResolver.purge << history_del).execute() logger.info('History cleaner Table DNSResolver delete row %d', count) for net in private_nets: ip_count = DNSResolver.delete().where(DNSResolver.ip % net).execute() if ip_count: logger.info('IP error LIKE %s, count %d', net, ip_count) History.update(resolver=True).where( History.id == self.code_id).execute() history_inconsist_sql = History.select().where( History.resolver == False) reslov_inconsist_count = DNSResolver.delete().where( DNSResolver.add << history_inconsist_sql).execute() DNSResolver.update(purge=None).where( DNSResolver.purge << history_inconsist_sql).execute() logger.info('Delete rows incomplete resolving process %d', reslov_inconsist_count)
def cleaner(self): private_nets = [ '0.%', '127.%', '192.168.%', '10.%', '172.16.%', '172.17.%', '172.18.%', '172.19.%', '172.20.%', '172.21.%', '172.22.%', '172.23.%', '172.24.%', '172.25.%', '172.26.%', '172.27.%', '172.28.%', '172.29.%', '172.30.%', '172.31.%' ] logger.info('Dump cleaner run') # history = History.select(History.id).order_by(History.id.desc()).limit(self.cfg.DiffCount()) # Item.delete().where(~(Item.purge << history)).execute() history_clear = History.select(History.id).order_by( History.id.desc()).offset(self.cfg.DiffCount()) item_del = Item.delete().where(Item.purge << history_clear).execute() logger.info('Item deleted: %d', item_del) ip_del = IP.delete().where(IP.purge << history_clear).execute() logger.info('IP deleted: %d', ip_del) domain_del = Domain.delete().where( Domain.purge << history_clear).execute() logger.info('Domain deleted: %d', domain_del) url_del = URL.delete().where(URL.purge << history_clear).execute() logger.info('URL deleted: %d', url_del) history_rm = History.select(History.id).order_by( History.id.desc()).offset(self.cfg.HistoryCount()) hist_del = History.delete().where(History.id << history_rm).execute() logger.info('History deleted: %d', hist_del) for net in private_nets: ip_count = IP.delete().where(IP.ip % net).execute() if ip_count: logger.info('IP error LIKE %s, count %d', net, ip_count)
def check_diff(self): idx_list = [ idx.id for idx in History.select(History.id).order_by( History.id.desc()).limit(self.cfg.DiffCount()) ] ip_diff_add_sql = IP.select(fn.Count(fn.Distinct( IP.ip))).join(Item).where(IP.add == idx_list[0]).scalar() ip_diff_purge_sql = IP.select(fn.Count(fn.Distinct( IP.ip))).join(Item).where(IP.purge == idx_list[0]).scalar() domain_diff_add_sql = Domain.select(fn.Count(fn.Distinct(Domain.domain)))\ .join(Item).where(Domain.add == idx_list[0]).scalar() domain_diff_purge_sql = Domain.select(fn.Count(fn.Distinct(Domain.domain)))\ .join(Item).where(Domain.purge == idx_list[0]).scalar() url_diff_add_sql = URL.select(fn.Count(fn.Distinct(URL.url)))\ .join(Item).where(URL.add == idx_list[0]).scalar() url_diff_purge_sql = URL.select(fn.Count(fn.Distinct(URL.url)))\ .join(Item).where(URL.purge == idx_list[0]).scalar() if ip_diff_add_sql or ip_diff_purge_sql or domain_diff_add_sql or \ domain_diff_purge_sql or url_diff_add_sql or url_diff_purge_sql: History.update(dump=True).where( History.id == idx_list[0]).execute() return True else: # History.update(dump=False).where(History.id == idx_list[0]).execute() return False
def cleaner(self): logger.info('cleaner run') # history = History.select(History.id).order_by(History.id.desc()).limit(self.cfg.DiffCount()) # Item.delete().where(~(Item.purge << history)).execute() history_del = History.select(History.id).order_by(History.id.desc()).offset(self.cfg.DiffCount()) Item.delete().where(Item.purge << history_del).execute() IP.delete().where(IP.purge << history_del).execute() Domain.delete().where(Domain.purge << history_del).execute() URL.delete().where(URL.purge << history_del).execute()
def check_diff(self): idx_list = [idx.id for idx in History.select(History.id).where(History.diff == True) .order_by(History.id.desc()).limit(self.cfg.DiffCount())] ip_diff_add_sql = IP.select(fn.Count(fn.Distinct(IP.ip))).join(Item).where(IP.add == idx_list[0]).scalar() ip_diff_purge_sql = IP.select(fn.Count(fn.Distinct(IP.ip))).join(Item).where(IP.purge == idx_list[0]).scalar() domain_diff_add_sql = Domain.select(fn.Count(fn.Distinct(Domain.domain)))\ .join(Item).where(Domain.add == idx_list[0]).scalar() domain_diff_purge_sql = Domain.select(fn.Count(fn.Distinct(Domain.domain)))\ .join(Item).where(Domain.purge == idx_list[0]).scalar() url_diff_add_sql = URL.select(fn.Count(fn.Distinct(URL.url)))\ .join(Item).where(URL.add == idx_list[0]).scalar() url_diff_purge_sql = URL.select(fn.Count(fn.Distinct(URL.url)))\ .join(Item).where(URL.purge == idx_list[0]).scalar() if ip_diff_add_sql or ip_diff_purge_sql or domain_diff_add_sql or \ domain_diff_purge_sql or url_diff_add_sql or url_diff_purge_sql: return True else: History.update(diff=False).where(History.id == idx_list[0]).execute() return False
def __init__(self, cfg): self.cfg = cfg self.idx_list = [idx.id for idx in History.select(History.id).where(History.diff == True) .order_by(History.id.desc()).limit(self.cfg.DiffCount())]
def history_show(): history_sql = History.select() for history_row in history_sql: print(history_row.date, history_row.requestCode)