def main(log_dir, server_name, database_spec, only=None, files=False):
sqlhub.processConnection = connectionForURI(database_spec)
if only:
domainlist = [only]
else:
domainlist = os.listdir(log_dir)
for domain in domainlist:
logger.info("Parsing logs for domain " + domain)
logfile = os.path.join(log_dir, domain, 'access.log')
p = Parser(files=files)
p.parse_file(logfile)
p.parse_file(logfile+'.1')
for date, d in p.valid_data().items():
if 'all' not in d:
continue
bytes, hits = d['all']
date_d = datetime.date(*[int(x) for x in date.split("-")])
try:
u = Usage.selectBy(date=date_d, domain=domain, server=server_name)[0]
except:
Usage(server=server_name, domain=domain, date=date,
bytes=bytes, hits=hits)
logger.debug("Added %s for domain %s" % (date, domain))
if date_d.strftime("%d.%m.%Y") == datetime.datetime.now().strftime("%d.%m.%Y"):
try:
u = Usage.selectBy(date=date_d, domain=domain, server=server_name)
for i in u:
u.delete(i.id)
except:
pass
d.pop('all')
for filename, e in d.items():
bytes, hits = e
date_d = datetime.date(*[int(x) for x in date.split("-")])
try:
u = UsageFiles.selectBy(date=date_d, domain=domain, server=server_name, filename=filename)[0]
continue
except:
UsageFiles(server=server_name, domain=domain, date=date,
bytes=bytes, hits=hits, filename = filename)
logger.debug("Added %s for domain %s file %s" % (date, domain, filename))
if date_d.strftime("%d.%m.%Y") == datetime.datetime.now().strftime("%d.%m.%Y"):
UsageFiles(server=server_name, domain=domain, date=date,
bytes=bytes, hits=hits, filename = filename)
logger.debug("Added %s for domain %s file %s" % (date, domain, filename))
def main(log_dir, server_name, db_url):
sqlhub.processConnection = connectionForURI(db_url)
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
for domain in os.listdir(log_dir):
logger.info("Parsing logs for domain " + domain)
logfile = os.path.join(log_dir, domain, 'access.log')
# p = Parser()
p = Parser(limit_to_date="%04i-%02i-%02i" % (yesterday.year, yesterday.month, yesterday.day))
p.parse_file(logfile)
p.parse_file(logfile+'.1')
for date, d in p.valid_data().iteritems():
bytes, hits = d
date_d = datetime.date(*[int(x) for x in date.split("-")])
try:
u = Usage.selectBy(date=date_d, domain=domain, server=server_name)[0]
continue
except:
Usage(server=server_name, domain=domain, date=date,
bytes=bytes, hits=hits)
logger.debug("Added %s for domain %s" % (date, domain))
