Example #1
0
    def load_reducers(self):
        self.reducers = {}

        if hasattr(self.config, 'REDUCERS'):
            for reducer_class in self.config.REDUCERS:
                reducer = kls_import(reducer_class)
                self.reducers[reducer.job_type] = reducer()
Example #2
0
    def load_input_streams(self):
        self.input_streams = {}

        if hasattr(self.config, 'INPUT_STREAMS'):
            for stream_class in self.config.INPUT_STREAMS:
                stream = kls_import(stream_class)
                self.input_streams[stream.job_type] = stream()
Example #3
0
File: app.py Project: Fangang/r3
    def load_reducers(self):
        self.reducers = {}

        if hasattr(self.config, 'REDUCERS'):
            for reducer_class in self.config.REDUCERS:
                reducer = kls_import(reducer_class)
                self.reducers[reducer.job_type] = reducer()
Example #4
0
File: app.py Project: Fangang/r3
    def load_input_streams(self):
        self.input_streams = {}

        if hasattr(self.config, 'INPUT_STREAMS'):
            for stream_class in self.config.INPUT_STREAMS:
                stream = kls_import(stream_class)
                self.input_streams[stream.job_type] = stream()
Example #5
0
def main(arguments=None):
    if not arguments:
        arguments = sys.argv[1:]

    parser = argparse.ArgumentParser(description='runs the application that processes stream requests for r³')
    parser.add_argument('-l', '--loglevel', type=str, default='warning', help='the log level that r³ will run under')
    parser.add_argument('--redis-host', type=str, default='0.0.0.0', help='the ip that r³ will use to connect to redis')
    parser.add_argument('--redis-port', type=int, default=6379, help='the port that r³ will use to connect to redis')
    parser.add_argument('--redis-db', type=int, default=0, help='the database that r³ will use to connect to redis')
    parser.add_argument('--redis-pass', type=str, default='', help='the password that r³ will use to connect to redis')
    parser.add_argument('--mapper-key', type=str, help='the unique identifier for this mapper', required=True)
    parser.add_argument('--mapper-class', type=str, help='the fullname of the class that this mapper will run', required=True)

    args = parser.parse_args(arguments)

    if not args.mapper_key:
        raise RuntimeError('The --mapper_key argument is required.')

    logging.basicConfig(level=getattr(logging, args.loglevel.upper()))

    try:
        klass = kls_import(args.mapper_class)
    except Exception, err:
        print "Could not import the specified %s class. Error: %s" % (args.mapper_class, err)
        raise
Example #6
0
def main(arguments=None):
    if not arguments:
        arguments = sys.argv[1:]

    parser = argparse.ArgumentParser(
        description='runs the application that processes stream requests for r³'
    )
    parser.add_argument('-l',
                        '--loglevel',
                        type=str,
                        default='warning',
                        help='the log level that r³ will run under')
    parser.add_argument('--redis-host',
                        type=str,
                        default='0.0.0.0',
                        help='the ip that r³ will use to connect to redis')
    parser.add_argument('--redis-port',
                        type=int,
                        default=6379,
                        help='the port that r³ will use to connect to redis')
    parser.add_argument(
        '--redis-db',
        type=int,
        default=0,
        help='the database that r³ will use to connect to redis')
    parser.add_argument(
        '--redis-pass',
        type=str,
        default='',
        help='the password that r³ will use to connect to redis')
    parser.add_argument('--mapper-key',
                        type=str,
                        help='the unique identifier for this mapper',
                        required=True)
    parser.add_argument(
        '--mapper-class',
        type=str,
        help='the fullname of the class that this mapper will run',
        required=True)

    args = parser.parse_args(arguments)

    if not args.mapper_key:
        raise RuntimeError('The --mapper_key argument is required.')

    logging.basicConfig(level=getattr(logging, args.loglevel.upper()))

    try:
        klass = kls_import(args.mapper_class)
    except Exception, err:
        print "Could not import the specified %s class. Error: %s" % (
            args.mapper_class, err)
        raise