def main(confdir="/etc/cslbot") -> None:
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
channel = '#tjhsst'
type_filter = or_(Log.type == 'privmsg', Log.type == 'pubmsg', Log.type == 'action')
users = session.query(Log.source).filter(Log.target == channel, type_filter).having(func.count(Log.id) > 500).group_by(Log.source).all()
freq = []
for user in users:
lines = session.query(Log.msg).filter(Log.target == channel, Log.source == user[0],
or_(Log.type == 'privmsg', Log.type == 'pubmsg', Log.type == 'action')).all()
text = '\n'.join([x[0] for x in lines])
with open('/tmp/foo', 'w') as f:
f.write(text)
try:
output = subprocess.check_output(['zpaq', 'add', 'foo.zpaq', '/tmp/foo', '-test', '-summary', '1', '-method', '5'],
stderr=subprocess.STDOUT,
universal_newlines=True)
sizes = output.splitlines()[-2]
match = re.match(r'.*\((.*) -> .* -> (.*)\).*', sizes)
if not match:
raise Exception('oh no')
before, after = match.groups()
# 8 bits = 1 byte
count = 1024 * 1024 * 8 * float(after) / len(text)
freq.append((user[0], len(lines), float(after) / float(before) * 100, count))
except subprocess.CalledProcessError as e:
print(e.stdout)
raise e
with open('freq.json', 'w') as f:
json.dump(freq, f, indent=True)
for x in sorted(freq, key=lambda x: x[2]):
print("%s: (%d lines) (%f%% compressed) (%f bits per char)" % x)
def real_main(argv) -> None:
if len(argv) > 1:
raise app.UsageError("Unexpected argument(s) received: %s" % argv)
# If we're running from a git checkout, override paths.
parent_directory = os.path.join(os.path.dirname(__file__), '..')
if os.path.exists(os.path.join(parent_directory, '.git')):
sys.path.insert(0, parent_directory)
FLAGS.set_default('confdir', parent_directory)
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(os.path.join(FLAGS.confdir, 'config.cfg')) as f:
config.read_file(f)
from cslbot.helpers import babble, sql
session = sql.get_session(config)()
cmdchar = config['core']['cmdchar']
ctrlchan = config['core']['ctrlchan']
print('Generating markov.')
# FIXME: support locking for other dialects?
if session.bind.dialect.name == 'postgresql':
session.execute('LOCK TABLE babble IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble2 IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_count IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_last IN EXCLUSIVE MODE NOWAIT')
t = time.time()
babble.build_markov(session,
cmdchar,
ctrlchan,
FLAGS.nick,
initial_run=not FLAGS.incremental,
debug=True)
print('Finished markov in %f' % (time.time() - t))
def main(argv) -> None:
if len(argv) > 1:
raise app.UsageError("Unexpected argument(s) received: %s" % argv)
# If we're running from a git checkout, override paths.
parent_directory = os.path.join(os.path.dirname(__file__), '..')
if os.path.exists(os.path.join(parent_directory, '.git')):
sys.path.insert(0, parent_directory)
FLAGS.set_default('confdir', parent_directory)
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(os.path.join(FLAGS.confdir, 'config.cfg')) as f:
config.read_file(f)
from cslbot.helpers import babble, sql
session = sql.get_session(config)()
cmdchar = config['core']['cmdchar']
ctrlchan = config['core']['ctrlchan']
print('Generating markov.')
# FIXME: support locking for other dialects?
if session.bind.dialect.name == 'postgresql':
session.execute('LOCK TABLE babble IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble2 IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_count IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_last IN EXCLUSIVE MODE NOWAIT')
t = time.time()
babble.build_markov(session, cmdchar, ctrlchan, FLAGS.nick, initial_run=not FLAGS.incremental, debug=True)
print('Finished markov in %f' % (time.time() - t))
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The output dir.')
cmdargs = parser.parse_args()
session = get_session(config)()
template_path = resource_filename(Requirement.parse('CslBot'), 'cslbot/templates')
env = Environment(loader=FileSystemLoader(template_path))
time = strftime('Last Updated at %I:%M %p on %a, %b %d, %Y')
if not path.exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open(path.join(cmdargs.outdir, '.lock'), 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Copy the js
shutil.copy(path.join(template_path, 'sorttable.js'), cmdargs.outdir)
output_quotes(env, session, cmdargs.outdir, time)
output_scores(env, session, cmdargs.outdir, time)
output_polls(env, session, cmdargs.outdir, time)
output_urls(env, session, cmdargs.outdir, time)
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The directory to write logs too.')
cmdargs = parser.parse_args()
if not exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open('%s/.lock' % cmdargs.outdir, 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
current_id = get_id(cmdargs.outdir)
new_id = session.query(Log.id).order_by(Log.id.desc()).limit(1).scalar()
# Don't die on empty log table.
if new_id is None:
new_id = 0
save_id(cmdargs.outdir, new_id)
for row in session.query(Log).filter(new_id >= Log.id).filter(Log.id > current_id).order_by(Log.time, Log.id).all():
check_day(row, cmdargs.outdir, config['core']['channel'])
write_log(row.target, cmdargs.outdir, gen_log(row))
for x in logs.values():
x.close()
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(join(confdir, 'config.cfg')) as f:
config.read_file(f)
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The output dir.')
cmdargs = parser.parse_args()
session = get_session(config)()
env = Environment(loader=FileSystemLoader(
resource_filename(Requirement.parse('CslBot'), 'cslbot/templates')))
time = strftime('Last Updated at %I:%M %p on %a, %b %d, %Y')
if not exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open('%s/.lock' % cmdargs.outdir, 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
output_quotes(env, session, cmdargs.outdir, time)
output_scores(env, session, cmdargs.outdir, time)
output_polls(env, session, cmdargs.outdir, time)
output_urls(env, session, cmdargs.outdir, time)
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(join(confdir, 'config.cfg')) as f:
config.read_file(f)
parser = argparse.ArgumentParser()
parser.add_argument(
'--nick', help='The nick to generate babble cache for (testing only).')
args = parser.parse_args()
session = sql.get_session(config)()
cmdchar = config['core']['cmdchar']
ctrlchan = config['core']['ctrlchan']
print('Generating markov.')
# FIXME: support locking for other dialects?
if session.bind.dialect.name == 'postgresql':
session.execute('LOCK TABLE babble IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_count IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_last IN EXCLUSIVE MODE NOWAIT')
t = time.time()
babble.build_markov(session,
cmdchar,
ctrlchan,
args.nick,
initial_run=True,
debug=True)
print('Finished markov in %f' % (time.time() - t))
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The directory to write logs too.')
cmdargs = parser.parse_args()
if not exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open('%s/.lock' % cmdargs.outdir, 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
current_id = get_id(cmdargs.outdir)
new_id = session.query(Log.id).order_by(Log.id.desc()).limit(1).scalar()
# Don't die on empty log table.
if new_id is None:
new_id = 0
save_id(cmdargs.outdir, new_id)
for row in session.query(Log).filter(new_id >= Log.id).filter(
Log.id > current_id).order_by(Log.time, Log.id).all():
check_day(row, cmdargs.outdir, config['core']['channel'])
write_log(row.target, cmdargs.outdir, gen_log(row))
for x in logs.values():
x.close()
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir="/etc/cslbot") -> None:
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The output dir.')
cmdargs = parser.parse_args()
session = get_session(config)()
time = strftime('Last Updated at %I:%M %p on %a, %b %d, %Y')
if not path.exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open(path.join(cmdargs.outdir, '.lock'), 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
template_path = resources.files('cslbot') / 'templates'
# Copy the js
shutil.copy(path.join(template_path, 'sorttable.js'), cmdargs.outdir)
env = Environment(loader=FileSystemLoader(template_path))
output_quotes(env, session, cmdargs.outdir, time)
output_scores(env, session, cmdargs.outdir, time)
output_polls(env, session, cmdargs.outdir, time)
output_urls(env, session, cmdargs.outdir, time)
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir="/etc/cslbot") -> None:
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
channel = '#tjhsst'
type_filter = or_(Log.type == 'privmsg', Log.type == 'pubmsg',
Log.type == 'action')
users = session.query(Log.source).filter(
Log.target == channel,
type_filter).having(func.count(Log.id) > 500).group_by(
Log.source).all()
freq = []
for user in users:
lines = session.query(Log.msg).filter(
Log.target == channel, Log.source == user[0],
or_(Log.type == 'privmsg', Log.type == 'pubmsg',
Log.type == 'action')).all()
text = '\n'.join([x[0] for x in lines])
with open('/tmp/foo', 'w') as f:
f.write(text)
try:
output = subprocess.check_output([
'zpaq', 'add', 'foo.zpaq', '/tmp/foo', '-test', '-summary',
'1', '-method', '5'
],
stderr=subprocess.STDOUT,
universal_newlines=True)
sizes = output.splitlines()[-2]
match = re.match(r'.*\((.*) -> .* -> (.*)\).*', sizes)
if not match:
raise Exception('oh no')
before, after = match.groups()
# 8 bits = 1 byte
count = 1024 * 1024 * 8 * float(after) / len(text)
freq.append((user[0], len(lines),
float(after) / float(before) * 100, count))
except subprocess.CalledProcessError as e:
print(e.stdout)
raise e
with open('freq.json', 'w') as f:
json.dump(freq, f, indent=True)
for x in sorted(freq, key=lambda x: x[2]):
print("%s: (%d lines) (%f%% compressed) (%f bits per char)" % x)
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(join(confdir, 'config.cfg')) as f:
config.read_file(f)
parser = argparse.ArgumentParser()
parser.add_argument('--nick', help='The nick to generate babble cache for (testing only).')
args = parser.parse_args()
session = sql.get_session(config)()
cmdchar = config['core']['cmdchar']
ctrlchan = config['core']['ctrlchan']
print('Generating markov.')
# FIXME: support locking for other dialects?
if session.bind.dialect.name == 'postgresql':
session.execute('LOCK TABLE babble IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_count IN EXCLUSIVE MODE NOWAIT')
session.execute('LOCK TABLE babble_last IN EXCLUSIVE MODE NOWAIT')
t = time.time()
babble.build_markov(session, cmdchar, ctrlchan, args.nick, initial_run=True, debug=True)
print('Finished markov in %f' % (time.time() - t))
def main(confdir="/etc/cslbot"):
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The directory to write logs too.')
cmdargs = parser.parse_args()
if not path.exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open(path.join(cmdargs.outdir, '.lock'), 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
current_id = get_id(cmdargs.outdir)
new_id = session.query(Log.id).order_by(Log.id.desc()).limit(1).scalar()
# Don't die on empty log table.
if new_id is None:
new_id = 0
save_id(cmdargs.outdir, new_id)
processer = LogProcesser(cmdargs.outdir)
for row in session.query(Log).filter(new_id >= Log.id).filter(Log.id > current_id).order_by(Log.time, Log.id).all():
processer.process_line(row)
del processer
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
def main(confdir: str = "/etc/cslbot") -> None:
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
with open(path.join(confdir, 'config.cfg')) as f:
config.read_file(f)
session = get_session(config)()
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help='The directory to write logs too.')
cmdargs = parser.parse_args()
if not path.exists(cmdargs.outdir):
makedirs(cmdargs.outdir)
lockfile = open(path.join(cmdargs.outdir, '.lock'), 'w')
fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
current_id = get_id(cmdargs.outdir)
new_id = session.query(Log.id).order_by(Log.id.desc()).limit(1).scalar()
# Don't die on empty log table.
if new_id is None:
new_id = 0
save_id(cmdargs.outdir, new_id)
processer = LogProcesser(cmdargs.outdir)
for row in session.query(Log).filter(new_id >= Log.id).filter(Log.id > current_id).order_by(Log.time, Log.id).all():
processer.process_line(row)
del processer
fcntl.lockf(lockfile, fcntl.LOCK_UN)
lockfile.close()
