def main(arguments=None):
if not arguments:
arguments = sys.argv[1:]
parser = argparse.ArgumentParser(description='runs the application that processes stream requests for r³')
parser.add_argument('-l', '--loglevel', type=str, default='warning', help='the log level that Surifki Refine will run under')
parser.add_argument('--redis-host', type=str, default='0.0.0.0', help='the ip that Surifki Refine will use to connect to redis')
parser.add_argument('--redis-port', type=int, default=6379, help='the port that Surifki Refine will use to connect to redis')
parser.add_argument('--redis-db', type=int, default=0, help='the database that Surifki Refine will use to connect to redis')
parser.add_argument('--redis-pass', type=str, default='', help='the password that Surifki Refine will use to connect to redis')
parser.add_argument('--mapper-key', type=str, help='the unique identifier for this mapper', required=True)
parser.add_argument('--mapper-class', type=str, help='the fullname of the class that this mapper will run', required=True)
args = parser.parse_args(arguments)
if not args.mapper_key:
raise RuntimeError('The --mapper_key argument is required.')
logging.basicConfig(level=getattr(logging, args.loglevel.upper()))
try:
klass = kls_import(args.mapper_class)
except Exception, err:
print "Could not import the specified %s class. Error: %s" % (args.mapper_class, err)
raise
def load_reducers(self):
self.reducers = {}
for reducer_class in self.config.REDUCERS:
reducer = kls_import(reducer_class)
self.reducers[reducer.job_type] = reducer()
def load_reducers(self):
self.reducers = {}
for reducer_class in self.config.REDUCERS:
reducer = kls_import(reducer_class)
self.reducers[reducer.job_type] = reducer()
def load_input_streams(self):
self.input_streams = {}
for stream_class in self.config.INPUT_STREAMS:
stream = kls_import(stream_class)
self.input_streams[stream.job_type] = stream()
def load_input_streams(self):
self.input_streams = {}
for stream_class in self.config.INPUT_STREAMS:
stream = kls_import(stream_class)
self.input_streams[stream.job_type] = stream()