def main(log_dir, server_name, database_spec, only=None, files=False): sqlhub.processConnection = connectionForURI(database_spec) if only: domainlist = [only] else: domainlist = os.listdir(log_dir) for domain in domainlist: logger.info("Parsing logs for domain " + domain) logfile = os.path.join(log_dir, domain, 'access.log') p = Parser(files=files) p.parse_file(logfile) p.parse_file(logfile+'.1') for date, d in p.valid_data().items(): if 'all' not in d: continue bytes, hits = d['all'] date_d = datetime.date(*[int(x) for x in date.split("-")]) try: u = Usage.selectBy(date=date_d, domain=domain, server=server_name)[0] except: Usage(server=server_name, domain=domain, date=date, bytes=bytes, hits=hits) logger.debug("Added %s for domain %s" % (date, domain)) if date_d.strftime("%d.%m.%Y") == datetime.datetime.now().strftime("%d.%m.%Y"): try: u = Usage.selectBy(date=date_d, domain=domain, server=server_name) for i in u: u.delete(i.id) except: pass d.pop('all') for filename, e in d.items(): bytes, hits = e date_d = datetime.date(*[int(x) for x in date.split("-")]) try: u = UsageFiles.selectBy(date=date_d, domain=domain, server=server_name, filename=filename)[0] continue except: UsageFiles(server=server_name, domain=domain, date=date, bytes=bytes, hits=hits, filename = filename) logger.debug("Added %s for domain %s file %s" % (date, domain, filename)) if date_d.strftime("%d.%m.%Y") == datetime.datetime.now().strftime("%d.%m.%Y"): UsageFiles(server=server_name, domain=domain, date=date, bytes=bytes, hits=hits, filename = filename) logger.debug("Added %s for domain %s file %s" % (date, domain, filename))
def main(log_dir, server_name, db_url): sqlhub.processConnection = connectionForURI(db_url) yesterday = datetime.datetime.now() - datetime.timedelta(days=1) for domain in os.listdir(log_dir): logger.info("Parsing logs for domain " + domain) logfile = os.path.join(log_dir, domain, 'access.log') # p = Parser() p = Parser(limit_to_date="%04i-%02i-%02i" % (yesterday.year, yesterday.month, yesterday.day)) p.parse_file(logfile) p.parse_file(logfile+'.1') for date, d in p.valid_data().iteritems(): bytes, hits = d date_d = datetime.date(*[int(x) for x in date.split("-")]) try: u = Usage.selectBy(date=date_d, domain=domain, server=server_name)[0] continue except: Usage(server=server_name, domain=domain, date=date, bytes=bytes, hits=hits) logger.debug("Added %s for domain %s" % (date, domain))