def detect(data, headers=None):
"""Return charset of data"""
from madcow.util import get_logger
log = get_logger()
# try to figure out the encoding first from meta tags
charset = metacharset(data)
if charset:
log.debug(u'using http meta header encoding: %s' % charset)
return charset
# if that doesnu't work, see if there's a real http header
if headers and headers.plist:
charset = headers.plist[0]
attrs = parseattrs(charset)
if u'charset' in attrs:
charset = lookup(attrs[u'charset'])
if charset:
log.debug(u'using http header encoding: %s' % charset)
return charset
# that didn't work, try chardet library
charset = lookup(chardet.detect(data)[u'encoding'])
if charset:
log.debug(u'detected encoding: %s' % repr(charset))
return charset
# if that managed to fail, just use ascii
log.warn(u"couldn't detect encoding, using ascii")
return DEFAULT
def detect(data, headers=None):
"""Return charset of data"""
from madcow.util import get_logger
log = get_logger()
# try to figure out the encoding first from meta tags
charset = metacharset(data)
if charset:
log.debug(u"using http meta header encoding: %s" % charset)
return charset
# if that doesnu't work, see if there's a real http header
if headers and headers.plist:
charset = headers.plist[0]
attrs = parseattrs(charset)
if u"charset" in attrs:
charset = lookup(attrs[u"charset"])
if charset:
log.debug(u"using http header encoding: %s" % charset)
return charset
# that didn't work, try chardet library
charset = lookup(chardet.detect(data)[u"encoding"])
if charset:
log.debug(u"detected encoding: %s" % repr(charset))
return charset
# if that managed to fail, just use ascii
log.warn(u"couldn't detect encoding, using ascii")
return DEFAULT
def run(base):
"""Execute the main bot"""
# if this is a new bot, create base and stop
base = os.path.realpath(base)
for subdir in 'db', 'log':
dir = os.path.join(base, subdir)
if not os.path.exists(dir):
os.makedirs(dir)
settings_file = os.path.join(base, 'settings.py')
default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
if not os.path.exists(settings_file):
shutil.copy(default_settings_file, settings_file)
os.chmod(settings_file, 0644)
raise MadcowError(
'A new bot has been created at %s, please modify config and rerun'
% base)
os.environ['MADCOW_BASE'] = base
log = get_logger('madcow', stream=sys.stdout, unique=False)
try:
log.info('setting up http')
http.setup(cookies=settings.HTTP_COOKIES,
agent=settings.HTTP_AGENT,
timeout=settings.HTTP_TIMEOUT)
log.info('loading protocol handler')
protocol = __import__('madcow.protocol', globals(), locals(),
[settings.PROTOCOL])
protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler
if settings.DETACH and protocol.allow_detach:
log.info('turning into a daemon')
daemonize()
if os.path.exists(settings.PIDFILE):
log.warn('removing stale pidfile')
os.remove(settings.PIDFILE)
with open(settings.PIDFILE, 'wb') as fp:
fp.write(str(os.getpid()))
protocol(base).start()
except:
log.exception('A fatal error ocurred')
raise
finally:
if os.path.exists(settings.PIDFILE):
log.info('removing pidfile')
os.remove(settings.PIDFILE)
def run(self):
"""While bot is alive, listen for connections"""
if not settings.GATEWAY_ENABLED:
return
self.log = get_logger('gateway', unique=False)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((settings.GATEWAY_ADDR, settings.GATEWAY_PORT))
sock.listen(5)
while self.bot.running:
client, addr = sock.accept()
self.log.info(u'connection from %s' % repr(addr))
handler = GatewayServiceHandler(self, client, addr)
handler.start()
def run(self):
"""While bot is alive, listen for connections"""
if not settings.GATEWAY_ENABLED:
return
self.log = get_logger('gateway', unique=False)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((settings.GATEWAY_ADDR, settings.GATEWAY_PORT))
sock.listen(5)
while self.bot.running:
client, addr = sock.accept()
self.log.info(u'connection from %s' % repr(addr))
handler = GatewayServiceHandler(self, client, addr)
handler.start()
def __init__(self, base, scheme=None):
"""Initialize bot"""
self.base = base
self.log = get_logger('madcow', unique=False, stream=sys.stdout)
self.colorlib = ColorLib(scheme)
self.cached_nick = None
self.running = False
self.ignore_list = [nick.lower() for nick in settings.IGNORE_NICKS]
# set encoding
set_encoding(settings.ENCODING)
self.log.info('Preferred encoding set to %s', get_encoding())
# create admin instance
self.admin = Admin(self)
# load modules
self.modules = Modules(self,
'modules',
settings.MODULES,
supercedes=settings.MODULE_SUPERCEDES)
self.tasks = Modules(self,
'tasks',
settings.TASKS,
supercedes=settings.TASK_SUPERCEDES)
self.usage_lines = self.modules.help + self.tasks.help
self.usage_lines.append(u'help - this screen')
self.usage_lines.append(u'version - get bot version')
# signal handlers
if signal:
signal.signal(signal.SIGHUP, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
# initialize threads
self.request_queue = queue.Queue()
self.response_queue = queue.Queue()
self.lock = threading.RLock()
self.ignore_res = []
if settings.IGNORE_REGEX:
for pattern in settings.IGNORE_REGEX:
try:
regex = re.compile(pattern, re.I)
except Exception, error:
self.log.warn('%r pattern did not compile', pattern)
continue
self.ignore_res.append(regex)
def run(base):
"""Execute the main bot"""
# if this is a new bot, create base and stop
base = os.path.realpath(base)
for subdir in 'db', 'log':
dir = os.path.join(base, subdir)
if not os.path.exists(dir):
os.makedirs(dir)
settings_file = os.path.join(base, 'settings.py')
default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
if not os.path.exists(settings_file):
shutil.copy(default_settings_file, settings_file)
os.chmod(settings_file, 0644)
raise MadcowError('A new bot has been created at %s, please modify config and rerun' % base)
os.environ['MADCOW_BASE'] = base
log = get_logger('madcow', stream=sys.stdout, unique=False)
try:
log.info('setting up http')
http.setup(cookies=settings.HTTP_COOKIES, agent=settings.HTTP_AGENT, timeout=settings.HTTP_TIMEOUT)
log.info('loading protocol handler')
protocol = __import__('madcow.protocol', globals(), locals(), [settings.PROTOCOL])
protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler
if settings.DETACH and protocol.allow_detach:
log.info('turning into a daemon')
daemonize()
if os.path.exists(settings.PIDFILE):
log.warn('removing stale pidfile')
os.remove(settings.PIDFILE)
with open(settings.PIDFILE, 'wb') as fp:
fp.write(str(os.getpid()))
protocol(base).start()
except:
log.exception('A fatal error ocurred')
raise
finally:
if os.path.exists(settings.PIDFILE):
log.info('removing pidfile')
os.remove(settings.PIDFILE)
def __init__(self, base, scheme=None):
"""Initialize bot"""
self.base = base
self.log = get_logger('madcow', unique=False, stream=sys.stdout)
self.colorlib = ColorLib(scheme)
self.cached_nick = None
self.running = False
self.ignore_list = [nick.lower() for nick in settings.IGNORE_NICKS]
# set encoding
set_encoding(settings.ENCODING)
self.log.info('Preferred encoding set to %s', get_encoding())
# create admin instance
self.admin = Admin(self)
# load modules
self.modules = Modules(self, 'modules', settings.MODULES)
self.tasks = Modules(self, 'tasks', settings.TASKS)
self.usage_lines = self.modules.help + self.tasks.help
self.usage_lines.append(u'help - this screen')
self.usage_lines.append(u'version - get bot version')
# signal handlers
if signal:
signal.signal(signal.SIGHUP, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
# initialize threads
self.request_queue = queue.Queue()
self.response_queue = queue.Queue()
self.lock = threading.RLock()
self.ignore_res = []
if settings.IGNORE_REGEX:
for pattern in settings.IGNORE_REGEX:
try:
regex = re.compile(pattern, re.I)
except Exception, error:
self.log.warn('%r pattern did not compile', pattern)
continue
self.ignore_res.append(regex)
def run(base, err=None, noeditor=False):
"""Execute the main bot"""
if err is None:
err = sys.stderr
# if this is a new bot, create base and stop
base = os.path.abspath(base)
for subdir in 'db', 'log':
dir = os.path.join(base, subdir)
if not os.path.exists(dir):
os.makedirs(dir)
settings_file = os.path.join(base, 'settings.py')
default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
if not os.path.exists(settings_file):
shutil.copy(default_settings_file, settings_file)
os.chmod(settings_file, 0640)
# TODO this code grew like a cancer. move it elsewhere, it's longer than the main loop..
if not noeditor:
# try to launch the editor configured in the user's environment instead
# of erroring out on first run, which is sort of rude.
try:
for env in 'EDITOR', 'VISUAL':
ed = os.environ.get(env)
if ed is not None:
# handle both path lookup and explicit path to program
if os.path.sep in ed:
if not os.access(ed, os.F_OK | os.X_OK):
ed = None
else:
path = os.environ.get('PATH', os.path.defpath)
for dir in path.split(os.path.pathsep):
bin = os.path.join(dir, ed)
if os.access(bin, os.F_OK | os.X_OK):
ed = bin
break
else:
ed = None
if ed is not None:
ed = os.path.abspath(ed)
break
else:
ed = None
if ed is not None:
# make sure the user is actually at a terminal before launching $EDITOR
# there might be a better way to do this..? but it seems to do the trick.
for fd in xrange(3):
if not os.isatty(fd):
break
else:
# NOTE: i am not sure this is technically correct, as it
# would fail when the user specifies any arguments in the
# EDTIOR env... i know PAGER actually evals() shell code in
# some cases, allowing complex expressions but i am not even
# gonna go there. if it becomes a problem, shlex.split may help.
pid = os.fork()
if pid == 0:
os.execl(ed, ed, settings_file)
os._exit(1)
ret = 255
while True:
wpid, wstatus = os.waitpid(pid, 0)
if pid == wpid:
if os.WIFSIGNALED(wstatus):
ret = -os.WTERMSIG(wstatus)
elif os.WIFEXITED(wstatus):
ret = os.WEXITSTATUS(wstatus)
break
if ret != os.EX_OK:
print >> err
print >> err, '{} exited with status: {:d}'.format(
ed, ret)
print >> err
print >> err, 'if you are done configuring it, you may now rerun the bot to launch it'
print >> err
return ret
# the number of ways the above can go wrong are too numerous to test for, don't bother user with it
except (SystemExit, KeyboardInterrupt):
raise
except:
import traceback
traceback.print_exc()
# if they made it this far, invoking an editor was not in the cards. do things old-fashioned.
print >> err
print >> err, 'A new bot has been created, please configure it by editing this file, then rerun:'
print >> err
print >> err, settings_file
print >> err
return 0
os.environ['MADCOW_BASE'] = base
log = get_logger('madcow', stream=sys.stdout, unique=False)
try:
log.info('setting up http')
http.setup(cookies=settings.HTTP_COOKIES,
agent=settings.HTTP_AGENT,
timeout=settings.HTTP_TIMEOUT)
log.info('loading protocol handler')
protocol = __import__('madcow.protocol', globals(), locals(),
[settings.PROTOCOL])
protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler
if settings.DETACH and protocol.allow_detach:
log.info('turning into a daemon')
try:
daemonize()
except (SystemExit, KeyboardInterrupt):
raise
except:
pass
if os.path.exists(settings.PIDFILE):
log.warn('removing stale pidfile')
os.remove(settings.PIDFILE)
with open(settings.PIDFILE, 'wb') as fp:
fp.write(str(os.getpid()))
protocol(base).start()
except SystemExit:
raise
except:
log.exception('A fatal error ocurred')
raise
finally:
try:
if os.path.exists(settings.PIDFILE):
log.info('removing pidfile')
os.remove(settings.PIDFILE)
except:
pass
def run(base, err=None, noeditor=False):
"""Execute the main bot"""
if err is None:
err = sys.stderr
# if this is a new bot, create base and stop
base = os.path.abspath(base)
for subdir in 'db', 'log':
dir = os.path.join(base, subdir)
if not os.path.exists(dir):
os.makedirs(dir)
settings_file = os.path.join(base, 'settings.py')
default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
if not os.path.exists(settings_file):
shutil.copy(default_settings_file, settings_file)
os.chmod(settings_file, 0640)
# TODO this code grew like a cancer. move it elsewhere, it's longer than the main loop..
if not noeditor:
# try to launch the editor configured in the user's environment instead
# of erroring out on first run, which is sort of rude.
try:
for env in 'EDITOR', 'VISUAL':
ed = os.environ.get(env)
if ed is not None:
# handle both path lookup and explicit path to program
if os.path.sep in ed:
if not os.access(ed, os.F_OK | os.X_OK):
ed = None
else:
path = os.environ.get('PATH', os.path.defpath)
for dir in path.split(os.path.pathsep):
bin = os.path.join(dir, ed)
if os.access(bin, os.F_OK | os.X_OK):
ed = bin
break
else:
ed = None
if ed is not None:
ed = os.path.abspath(ed)
break
else:
ed = None
if ed is not None:
# make sure the user is actually at a terminal before launching $EDITOR
# there might be a better way to do this..? but it seems to do the trick.
for fd in xrange(3):
if not os.isatty(fd):
break
else:
# NOTE: i am not sure this is technically correct, as it
# would fail when the user specifies any arguments in the
# EDTIOR env... i know PAGER actually evals() shell code in
# some cases, allowing complex expressions but i am not even
# gonna go there. if it becomes a problem, shlex.split may help.
pid = os.fork()
if pid == 0:
os.execl(ed, ed, settings_file)
os._exit(1)
ret = 255
while True:
wpid, wstatus = os.waitpid(pid, 0)
if pid == wpid:
if os.WIFSIGNALED(wstatus):
ret = -os.WTERMSIG(wstatus)
elif os.WIFEXITED(wstatus):
ret = os.WEXITSTATUS(wstatus)
break
if ret != os.EX_OK:
print >> err
print >> err, '{} exited with status: {:d}'.format(ed, ret)
print >> err
print >> err, 'if you are done configuring it, you may now rerun the bot to launch it'
print >> err
return ret
# the number of ways the above can go wrong are too numerous to test for, don't bother user with it
except (SystemExit, KeyboardInterrupt):
raise
except:
import traceback
traceback.print_exc()
# if they made it this far, invoking an editor was not in the cards. do things old-fashioned.
print >> err
print >> err, 'A new bot has been created, please configure it by editing this file, then rerun:'
print >> err
print >> err, settings_file
print >> err
return 0
os.environ['MADCOW_BASE'] = base
log = get_logger('madcow', stream=sys.stdout, unique=False)
try:
log.info('setting up http')
http.setup(cookies=settings.HTTP_COOKIES, agent=settings.HTTP_AGENT, timeout=settings.HTTP_TIMEOUT)
log.info('loading protocol handler')
protocol = __import__('madcow.protocol', globals(), locals(), [settings.PROTOCOL])
protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler
if settings.DETACH and protocol.allow_detach:
log.info('turning into a daemon')
try:
daemonize()
except (SystemExit, KeyboardInterrupt):
raise
except:
pass
if os.path.exists(settings.PIDFILE):
log.warn('removing stale pidfile')
os.remove(settings.PIDFILE)
with open(settings.PIDFILE, 'wb') as fp:
fp.write(str(os.getpid()))
protocol(base).start()
except SystemExit:
raise
except:
log.exception('A fatal error ocurred')
raise
finally:
try:
if os.path.exists(settings.PIDFILE):
log.info('removing pidfile')
os.remove(settings.PIDFILE)
except:
pass
