Example #1
0
def detect(data, headers=None):
    """Return charset of data"""
    from madcow.util import get_logger
    log = get_logger()

    # try to figure out the encoding first from meta tags
    charset = metacharset(data)
    if charset:
        log.debug(u'using http meta header encoding: %s' % charset)
        return charset

    # if that doesnu't work, see if there's a real http header
    if headers and headers.plist:
        charset = headers.plist[0]
        attrs = parseattrs(charset)
        if u'charset' in attrs:
            charset = lookup(attrs[u'charset'])
        if charset:
            log.debug(u'using http header encoding: %s' % charset)
            return charset

    # that didn't work, try chardet library
    charset = lookup(chardet.detect(data)[u'encoding'])
    if charset:
        log.debug(u'detected encoding: %s' % repr(charset))
        return charset

    # if that managed to fail, just use ascii
    log.warn(u"couldn't detect encoding, using ascii")
    return DEFAULT
def detect(data, headers=None):
    """Return charset of data"""
    from madcow.util import get_logger

    log = get_logger()

    # try to figure out the encoding first from meta tags
    charset = metacharset(data)
    if charset:
        log.debug(u"using http meta header encoding: %s" % charset)
        return charset

    # if that doesnu't work, see if there's a real http header
    if headers and headers.plist:
        charset = headers.plist[0]
        attrs = parseattrs(charset)
        if u"charset" in attrs:
            charset = lookup(attrs[u"charset"])
        if charset:
            log.debug(u"using http header encoding: %s" % charset)
            return charset

    # that didn't work, try chardet library
    charset = lookup(chardet.detect(data)[u"encoding"])
    if charset:
        log.debug(u"detected encoding: %s" % repr(charset))
        return charset

    # if that managed to fail, just use ascii
    log.warn(u"couldn't detect encoding, using ascii")
    return DEFAULT
Example #3
0
def run(base):
    """Execute the main bot"""

    # if this is a new bot, create base and stop
    base = os.path.realpath(base)
    for subdir in 'db', 'log':
        dir = os.path.join(base, subdir)
        if not os.path.exists(dir):
            os.makedirs(dir)
    settings_file = os.path.join(base, 'settings.py')
    default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
    if not os.path.exists(settings_file):
        shutil.copy(default_settings_file, settings_file)
        os.chmod(settings_file, 0644)
        raise MadcowError(
            'A new bot has been created at %s, please modify config and rerun'
            % base)

    os.environ['MADCOW_BASE'] = base
    log = get_logger('madcow', stream=sys.stdout, unique=False)

    try:
        log.info('setting up http')
        http.setup(cookies=settings.HTTP_COOKIES,
                   agent=settings.HTTP_AGENT,
                   timeout=settings.HTTP_TIMEOUT)

        log.info('loading protocol handler')
        protocol = __import__('madcow.protocol', globals(), locals(),
                              [settings.PROTOCOL])
        protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler

        if settings.DETACH and protocol.allow_detach:
            log.info('turning into a daemon')
            daemonize()

        if os.path.exists(settings.PIDFILE):
            log.warn('removing stale pidfile')
            os.remove(settings.PIDFILE)
        with open(settings.PIDFILE, 'wb') as fp:
            fp.write(str(os.getpid()))

        protocol(base).start()

    except:
        log.exception('A fatal error ocurred')
        raise

    finally:
        if os.path.exists(settings.PIDFILE):
            log.info('removing pidfile')
            os.remove(settings.PIDFILE)
Example #4
0
 def run(self):
     """While bot is alive, listen for connections"""
     if not settings.GATEWAY_ENABLED:
         return
     self.log = get_logger('gateway', unique=False)
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     sock.bind((settings.GATEWAY_ADDR, settings.GATEWAY_PORT))
     sock.listen(5)
     while self.bot.running:
         client, addr = sock.accept()
         self.log.info(u'connection from %s' % repr(addr))
         handler = GatewayServiceHandler(self, client, addr)
         handler.start()
Example #5
0
 def run(self):
     """While bot is alive, listen for connections"""
     if not settings.GATEWAY_ENABLED:
         return
     self.log = get_logger('gateway', unique=False)
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     sock.bind((settings.GATEWAY_ADDR, settings.GATEWAY_PORT))
     sock.listen(5)
     while self.bot.running:
         client, addr = sock.accept()
         self.log.info(u'connection from %s' % repr(addr))
         handler = GatewayServiceHandler(self, client, addr)
         handler.start()
Example #6
0
    def __init__(self, base, scheme=None):
        """Initialize bot"""
        self.base = base
        self.log = get_logger('madcow', unique=False, stream=sys.stdout)
        self.colorlib = ColorLib(scheme)
        self.cached_nick = None
        self.running = False

        self.ignore_list = [nick.lower() for nick in settings.IGNORE_NICKS]

        # set encoding
        set_encoding(settings.ENCODING)
        self.log.info('Preferred encoding set to %s', get_encoding())

        # create admin instance
        self.admin = Admin(self)

        # load modules
        self.modules = Modules(self,
                               'modules',
                               settings.MODULES,
                               supercedes=settings.MODULE_SUPERCEDES)
        self.tasks = Modules(self,
                             'tasks',
                             settings.TASKS,
                             supercedes=settings.TASK_SUPERCEDES)
        self.usage_lines = self.modules.help + self.tasks.help
        self.usage_lines.append(u'help - this screen')
        self.usage_lines.append(u'version - get bot version')

        # signal handlers
        if signal:
            signal.signal(signal.SIGHUP, self.signal_handler)
            signal.signal(signal.SIGTERM, self.signal_handler)

        # initialize threads
        self.request_queue = queue.Queue()
        self.response_queue = queue.Queue()
        self.lock = threading.RLock()
        self.ignore_res = []
        if settings.IGNORE_REGEX:
            for pattern in settings.IGNORE_REGEX:
                try:
                    regex = re.compile(pattern, re.I)
                except Exception, error:
                    self.log.warn('%r pattern did not compile', pattern)
                    continue
                self.ignore_res.append(regex)
Example #7
0
def run(base):
    """Execute the main bot"""

    # if this is a new bot, create base and stop
    base = os.path.realpath(base)
    for subdir in 'db', 'log':
        dir = os.path.join(base, subdir)
        if not os.path.exists(dir):
            os.makedirs(dir)
    settings_file = os.path.join(base, 'settings.py')
    default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
    if not os.path.exists(settings_file):
        shutil.copy(default_settings_file, settings_file)
        os.chmod(settings_file, 0644)
        raise MadcowError('A new bot has been created at %s, please modify config and rerun' % base)

    os.environ['MADCOW_BASE'] = base
    log = get_logger('madcow', stream=sys.stdout, unique=False)

    try:
        log.info('setting up http')
        http.setup(cookies=settings.HTTP_COOKIES, agent=settings.HTTP_AGENT, timeout=settings.HTTP_TIMEOUT)

        log.info('loading protocol handler')
        protocol = __import__('madcow.protocol', globals(), locals(), [settings.PROTOCOL])
        protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler

        if settings.DETACH and protocol.allow_detach:
            log.info('turning into a daemon')
            daemonize()

        if os.path.exists(settings.PIDFILE):
            log.warn('removing stale pidfile')
            os.remove(settings.PIDFILE)
        with open(settings.PIDFILE, 'wb') as fp:
            fp.write(str(os.getpid()))

        protocol(base).start()

    except:
        log.exception('A fatal error ocurred')
        raise

    finally:
        if os.path.exists(settings.PIDFILE):
            log.info('removing pidfile')
            os.remove(settings.PIDFILE)
Example #8
0
    def __init__(self, base, scheme=None):
        """Initialize bot"""
        self.base = base
        self.log = get_logger('madcow', unique=False, stream=sys.stdout)
        self.colorlib = ColorLib(scheme)
        self.cached_nick = None
        self.running = False

        self.ignore_list = [nick.lower() for nick in settings.IGNORE_NICKS]

        # set encoding
        set_encoding(settings.ENCODING)
        self.log.info('Preferred encoding set to %s', get_encoding())

        # create admin instance
        self.admin = Admin(self)

        # load modules
        self.modules = Modules(self, 'modules', settings.MODULES)
        self.tasks = Modules(self, 'tasks', settings.TASKS)
        self.usage_lines = self.modules.help + self.tasks.help
        self.usage_lines.append(u'help - this screen')
        self.usage_lines.append(u'version - get bot version')

        # signal handlers
        if signal:
            signal.signal(signal.SIGHUP, self.signal_handler)
            signal.signal(signal.SIGTERM, self.signal_handler)

        # initialize threads
        self.request_queue = queue.Queue()
        self.response_queue = queue.Queue()
        self.lock = threading.RLock()
        self.ignore_res = []
        if settings.IGNORE_REGEX:
            for pattern in settings.IGNORE_REGEX:
                try:
                    regex = re.compile(pattern, re.I)
                except Exception, error:
                    self.log.warn('%r pattern did not compile', pattern)
                    continue
                self.ignore_res.append(regex)
Example #9
0
def run(base, err=None, noeditor=False):
    """Execute the main bot"""
    if err is None:
        err = sys.stderr

    # if this is a new bot, create base and stop
    base = os.path.abspath(base)
    for subdir in 'db', 'log':
        dir = os.path.join(base, subdir)
        if not os.path.exists(dir):
            os.makedirs(dir)
    settings_file = os.path.join(base, 'settings.py')
    default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
    if not os.path.exists(settings_file):
        shutil.copy(default_settings_file, settings_file)
        os.chmod(settings_file, 0640)

        # TODO this code grew like a cancer. move it elsewhere, it's longer than the main loop..
        if not noeditor:
            # try to launch the editor configured in the user's environment instead
            # of erroring out on first run, which is sort of rude.
            try:
                for env in 'EDITOR', 'VISUAL':
                    ed = os.environ.get(env)
                    if ed is not None:
                        # handle both path lookup and explicit path to program
                        if os.path.sep in ed:
                            if not os.access(ed, os.F_OK | os.X_OK):
                                ed = None
                        else:
                            path = os.environ.get('PATH', os.path.defpath)
                            for dir in path.split(os.path.pathsep):
                                bin = os.path.join(dir, ed)
                                if os.access(bin, os.F_OK | os.X_OK):
                                    ed = bin
                                    break
                            else:
                                ed = None
                        if ed is not None:
                            ed = os.path.abspath(ed)
                            break
                else:
                    ed = None

                if ed is not None:
                    # make sure the user is actually at a terminal before launching $EDITOR
                    # there might be a better way to do this..? but it seems to do the trick.
                    for fd in xrange(3):
                        if not os.isatty(fd):
                            break
                    else:
                        # NOTE: i am not sure this is technically correct, as it
                        # would fail when the user specifies any arguments in the
                        # EDTIOR env... i know PAGER actually evals() shell code in
                        # some cases, allowing complex expressions but i am not even
                        # gonna go there. if it becomes a problem, shlex.split may help.
                        pid = os.fork()
                        if pid == 0:
                            os.execl(ed, ed, settings_file)
                            os._exit(1)
                        ret = 255
                        while True:
                            wpid, wstatus = os.waitpid(pid, 0)
                            if pid == wpid:
                                if os.WIFSIGNALED(wstatus):
                                    ret = -os.WTERMSIG(wstatus)
                                elif os.WIFEXITED(wstatus):
                                    ret = os.WEXITSTATUS(wstatus)
                                break
                        if ret != os.EX_OK:
                            print >> err
                            print >> err, '{} exited with status: {:d}'.format(
                                ed, ret)

                        print >> err
                        print >> err, 'if you are done configuring it, you may now rerun the bot to launch it'
                        print >> err
                        return ret

            # the number of ways the above can go wrong are too numerous to test for, don't bother user with it
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                import traceback
                traceback.print_exc()

        # if they made it this far, invoking an editor was not in the cards. do things old-fashioned.
        print >> err
        print >> err, 'A new bot has been created, please configure it by editing this file, then rerun:'
        print >> err
        print >> err, settings_file
        print >> err
        return 0

    os.environ['MADCOW_BASE'] = base
    log = get_logger('madcow', stream=sys.stdout, unique=False)

    try:
        log.info('setting up http')
        http.setup(cookies=settings.HTTP_COOKIES,
                   agent=settings.HTTP_AGENT,
                   timeout=settings.HTTP_TIMEOUT)

        log.info('loading protocol handler')
        protocol = __import__('madcow.protocol', globals(), locals(),
                              [settings.PROTOCOL])
        protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler

        if settings.DETACH and protocol.allow_detach:
            log.info('turning into a daemon')
            try:
                daemonize()
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                pass

        if os.path.exists(settings.PIDFILE):
            log.warn('removing stale pidfile')
            os.remove(settings.PIDFILE)
        with open(settings.PIDFILE, 'wb') as fp:
            fp.write(str(os.getpid()))

        protocol(base).start()

    except SystemExit:
        raise
    except:
        log.exception('A fatal error ocurred')
        raise
    finally:
        try:
            if os.path.exists(settings.PIDFILE):
                log.info('removing pidfile')
                os.remove(settings.PIDFILE)
        except:
            pass
Example #10
0
def run(base, err=None, noeditor=False):
    """Execute the main bot"""
    if err is None:
        err = sys.stderr

    # if this is a new bot, create base and stop
    base = os.path.abspath(base)
    for subdir in 'db', 'log':
        dir = os.path.join(base, subdir)
        if not os.path.exists(dir):
            os.makedirs(dir)
    settings_file = os.path.join(base, 'settings.py')
    default_settings_file = os.path.join(PREFIX, 'conf', 'defaults.py')
    if not os.path.exists(settings_file):
        shutil.copy(default_settings_file, settings_file)
        os.chmod(settings_file, 0640)

        # TODO this code grew like a cancer. move it elsewhere, it's longer than the main loop..
        if not noeditor:
            # try to launch the editor configured in the user's environment instead
            # of erroring out on first run, which is sort of rude.
            try:
                for env in 'EDITOR', 'VISUAL':
                    ed = os.environ.get(env)
                    if ed is not None:
                        # handle both path lookup and explicit path to program
                        if os.path.sep in ed:
                            if not os.access(ed, os.F_OK | os.X_OK):
                                ed = None
                        else:
                            path = os.environ.get('PATH', os.path.defpath)
                            for dir in path.split(os.path.pathsep):
                                bin = os.path.join(dir, ed)
                                if os.access(bin, os.F_OK | os.X_OK):
                                    ed = bin
                                    break
                            else:
                                ed = None
                        if ed is not None:
                            ed = os.path.abspath(ed)
                            break
                else:
                    ed = None

                if ed is not None:
                    # make sure the user is actually at a terminal before launching $EDITOR
                    # there might be a better way to do this..? but it seems to do the trick.
                    for fd in xrange(3):
                        if not os.isatty(fd):
                            break
                    else:
                        # NOTE: i am not sure this is technically correct, as it
                        # would fail when the user specifies any arguments in the
                        # EDTIOR env... i know PAGER actually evals() shell code in
                        # some cases, allowing complex expressions but i am not even
                        # gonna go there. if it becomes a problem, shlex.split may help.
                        pid = os.fork()
                        if pid == 0:
                            os.execl(ed, ed, settings_file)
                            os._exit(1)
                        ret = 255
                        while True:
                            wpid, wstatus = os.waitpid(pid, 0)
                            if pid == wpid:
                                if os.WIFSIGNALED(wstatus):
                                    ret = -os.WTERMSIG(wstatus)
                                elif os.WIFEXITED(wstatus):
                                    ret = os.WEXITSTATUS(wstatus)
                                break
                        if ret != os.EX_OK:
                            print >> err
                            print >> err, '{} exited with status: {:d}'.format(ed, ret)

                        print >> err
                        print >> err, 'if you are done configuring it, you may now rerun the bot to launch it'
                        print >> err
                        return ret

            # the number of ways the above can go wrong are too numerous to test for, don't bother user with it
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                import traceback
                traceback.print_exc()

        # if they made it this far, invoking an editor was not in the cards. do things old-fashioned.
        print >> err
        print >> err, 'A new bot has been created, please configure it by editing this file, then rerun:'
        print >> err
        print >> err, settings_file
        print >> err
        return 0

    os.environ['MADCOW_BASE'] = base
    log = get_logger('madcow', stream=sys.stdout, unique=False)

    try:
        log.info('setting up http')
        http.setup(cookies=settings.HTTP_COOKIES, agent=settings.HTTP_AGENT, timeout=settings.HTTP_TIMEOUT)

        log.info('loading protocol handler')
        protocol = __import__('madcow.protocol', globals(), locals(), [settings.PROTOCOL])
        protocol = getattr(protocol, settings.PROTOCOL).ProtocolHandler

        if settings.DETACH and protocol.allow_detach:
            log.info('turning into a daemon')
            try:
                daemonize()
            except (SystemExit, KeyboardInterrupt):
                raise
            except:
                pass

        if os.path.exists(settings.PIDFILE):
            log.warn('removing stale pidfile')
            os.remove(settings.PIDFILE)
        with open(settings.PIDFILE, 'wb') as fp:
            fp.write(str(os.getpid()))

        protocol(base).start()

    except SystemExit:
        raise
    except:
        log.exception('A fatal error ocurred')
        raise
    finally:
        try:
            if os.path.exists(settings.PIDFILE):
                log.info('removing pidfile')
                os.remove(settings.PIDFILE)
        except:
            pass