def main():
db = HistDB()
cfg = ConfigParser()
cfg.read(config_filename)
db.connect(cfg.get("MySQL", "host"), cfg.get("MySQL", "user"), cfg.get("MySQL", "pass"), cfg.get("MySQL", "db"))
db.cleanup(cfg.get("Maintenance", "forget_files_after"))
def main(argv):
hdb = HistDB()
adb = AggDB()
cfg = ConfigParser()
cfg.read(hconfig_filename)
hdb.connect(cfg.get("MySQL", "host"),
cfg.get("MySQL", "user"),
cfg.get("MySQL", "pass"),
cfg.get("MySQL", "db"))
cfg.read(aconfig_filename)
adb.connect(cfg.get("MySQL", "host"),
cfg.get("MySQL", "user"),
cfg.get("MySQL", "pass"),
cfg.get("MySQL", "db"))
logging.config.fileConfig(config_logging_filename)
logger = logging.getLogger("file")
# All history:
for item in hdb.aggregate_history():
adb.addlease(item)
if item.complete:
hdb.del_history_records(item.rows)
logger.debug(str(item.rows) + " IDs deleted from history")
nr = adb.cleanup(cfg.get("Maintenance", "record_keep_days"))
logger.info(str(nr) + " leases deleted from aggregated")
def __init__(self, config):
self.server = platform.node()
self._db = HistDB()
self._cfg = ConfigParser.ConfigParser()
self._cfg.read(config)
self.timezone = pytz.timezone(self._cfg.get("App", "timezone"))
logging.config.fileConfig(config_logging_filename)
self._logger = logging.getLogger("file")
self._db.connect(
self._cfg.get("MySQL", "host"),
self._cfg.get("MySQL", "user"),
self._cfg.get("MySQL", "pass"),
self._cfg.get("MySQL", "db"),
)
self._logger.debug("init")
class HistFileImporter:
def __init__(self, config):
self.server = platform.node()
self._db = HistDB()
self._cfg = ConfigParser.ConfigParser()
self._cfg.read(config)
self.timezone = pytz.timezone(self._cfg.get("App", "timezone"))
logging.config.fileConfig(config_logging_filename)
self._logger = logging.getLogger("file")
self._db.connect(
self._cfg.get("MySQL", "host"),
self._cfg.get("MySQL", "user"),
self._cfg.get("MySQL", "pass"),
self._cfg.get("MySQL", "db"),
)
self._logger.debug("init")
def add(self, filenames):
"""Add file(s) to database"""
self._db.addfiles(self.server, filenames)
self._logger.debug("add" + " ".join(filenames))
def remove(self, filename):
"""Remove file from database"""
self._db.removefile(self.server, filename)
self._logger.warning(" ".join(["remove", filename]))
def process(self):
"""Process unprocessed files and import parsed data to database"""
for logfile in self._db.get_unprocessed_files(self.server):
try:
self.parse(logfile)
except IOError:
self.remove(self.server, logfile)
self._logger.debug("process")
def _tidy_dhcpd_hexstring(self, hexstr):
"""Take hex-encoded string in form a:bc:de:f
and transform it to 0abcde0f"""
return "".join(map(lambda x: x.rjust(2, "0"), hexstr.split(":")))
def _parse_syslog_timestamp(self, timestamp, year=None):
"""Parses Syslog format timestamps (Example: "Jan 16 13:12:24")"""
now = datetime.datetime.now()
if year is None:
current_year = now.year
else:
current_year = year
full_date = " ".join([str(current_year), timestamp])
ts = datetime.datetime.strptime(full_date, "%Y %b %d %H:%M:%S")
# Date has to be in the past:
if ts <= now:
return self.timezone.localize(ts)
else:
return self._parse_syslog_timestamp(timestamp, current_year - 1)
def _hexstring2printable(self, hexstr):
"""Outputs printable version of string"""
hs = self._tidy_dhcpd_hexstring(hexstr)
return hs.decode("hex").encode("string_escape")
def _parse_and_save_record(self, parts):
"""Processes log record and saves structured data to DB"""
if len(parts) > 5:
ts = self._parse_syslog_timestamp(" ".join(parts[0:3]))
# Always store time in UTC
utc_ts = ts.astimezone(pytz.utc)
timestamp = utc_ts.strftime("%Y-%m-%d %H:%M:%S")
ip_addr = parts[6]
state = None
if parts[5] == "LEASECOMMIT":
mac_addr = self._tidy_dhcpd_hexstring(parts[7])
circuit_id = self._hexstring2printable(parts[8])
remote_id = self._hexstring2printable(parts[9])
giaddr = parts[10]
lease_time = int(self._tidy_dhcpd_hexstring(parts[11]), 16)
state = "LEASED"
if parts[5] == "LEASERELEASE":
mac_addr = None
circuit_id = None
remote_id = None
giaddr = None
lease_time = None
state = "FREE"
if parts[5] == "LEASEEXPIRY":
mac_addr = None
circuit_id = None
remote_id = None
giaddr = None
lease_time = None
state = "FREE"
if not state is None:
try:
self._db.add_history_record(
timestamp, ip_addr, mac_addr, circuit_id, remote_id, giaddr, lease_time, state
)
except DBException as e:
self._logger.warning(e)
def parse(self, logfile):
"""Parse file and import data to database"""
with open(logfile, "r") as f:
for line in f:
# regexp is needed here because there can be
# multiple spaces between fields
parts = re.split("\s+", line.rstrip("\n"))
self._parse_and_save_record(parts)
self._db.mark_as_processed(self.server, logfile)
self._logger.debug(" ".join(["parse", logfile]))