def start(self):
if self.started:
return
self.init()
env.start(True, isLocal=self.isLocal)
self.scheduler.start()
self.started = True
atexit.register(self.stop)
def handler(signm, frame):
logger.error("got signal %d, exit now", signm)
self.scheduler.shutdown()
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
try:
from rfoo.utils import rconsole
rconsole.spawn_server(locals(), 0)
except ImportError:
pass
def start(self):
if self.started:
return
self.init()
env.start(True, isLocal=self.isLocal)
self.scheduler.start()
self.started = True
atexit.register(self.stop)
def handler(signm, frame):
logger.error("got signal %d, exit now", signm)
self.scheduler.shutdown()
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
try:
from rfoo.utils import rconsole
rconsole.spawn_server(locals(), 0)
except ImportError:
pass
def test():
l = []
for i in range(10):
d = zip(range(10000), range(10000))
l.append(sorted_items(d))
hl = heap_merged(l, lambda x,y:x+y)
for i in range(10):
print i, hl.next()
import logging
logging.basicConfig(level=logging.INFO)
from env import env
import cPickle
env.start(True)
path = LocalFileShuffle.getOutputFile(1, 0, 0)
f = open(path, 'w')
f.write(cPickle.dumps([('key','value')], -1))
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutput(1, 1, 0, uri)
fetcher = SimpleShuffleFetcher()
def func(k,v):
assert k=='key'
assert v=='value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker(True)
tracker.registerMapOutput(2, 5, 1, uri)
assert tracker.getServerUris(2) == [None, uri, None, None, None]
ntracker = MapOutputTracker(False)
assert ntracker.getServerUris(2) == [None, uri, None, None, None]
ntracker.stop()
tracker.stop()
def test():
import logging
logging.basicConfig(level=logging.INFO)
from env import env
import cPickle
env.start(True)
path = LocalFileShuffle.getOutputFile(1, 0, 0)
f = open(path, 'w')
f.write(cPickle.dumps([('key','value')], -1))
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutput(1, 1, 0, uri)
fetcher = SimpleShuffleFetcher()
def func(k,v):
assert k=='key'
assert v=='value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker(True)
tracker.registerMapOutput(2, 5, 1, uri)
assert tracker.getServerUris(2) == [None, uri, None, None, None]
ntracker = MapOutputTracker(False)
assert ntracker.getServerUris(2) == [None, uri, None, None, None]
ntracker.stop()
tracker.stop()
def test():
l = []
for i in range(10):
d = zip(range(10000), range(10000))
l.append(sorted_items(d))
hl = heap_merged(l, lambda x,y:x+y)
for i in range(10):
print i, hl.next()
import logging
logging.basicConfig(level=logging.INFO)
from env import env
import cPickle
env.start(True)
path = LocalFileShuffle.getOutputFile(1, 0, 0)
f = open(path, 'w')
f.write(cPickle.dumps([('key','value')], -1))
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutput(1, 1, 0, uri)
fetcher = SimpleShuffleFetcher()
def func(k,v):
assert k=='key'
assert v=='value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker(True)
tracker.registerMapOutput(2, 5, 1, uri)
assert tracker.getServerUris(2) == [None, uri, None, None, None]
ntracker = MapOutputTracker(False)
assert ntracker.getServerUris(2) == [None, uri, None, None, None]
ntracker.stop()
tracker.stop()
def test():
import logging
logging.basicConfig(level=logging.INFO)
from env import env
import cPickle
env.start(True)
path = LocalFileShuffle.getOutputFile(1, 0, 0)
f = open(path, 'w')
f.write(cPickle.dumps([('key','value')], -1))
f.close()
uri = LocalFileShuffle.getServerUri()
env.mapOutputTracker.registerMapOutput(1, 1, 0, uri)
fetcher = SimpleShuffleFetcher()
def func(k,v):
assert k=='key'
assert v=='value'
fetcher.fetch(1, 0, func)
tracker = MapOutputTracker(True)
tracker.registerMapOutput(2, 5, 1, uri)
assert tracker.getServerUris(2) == [None, uri, None, None, None]
ntracker = MapOutputTracker(False)
assert ntracker.getServerUris(2) == [None, uri, None, None, None]
ntracker.stop()
tracker.stop()
def run_task_in_process(task, tid, environ):
from env import env
env.start(False, environ)
logger.debug("run task in process %s %s", task, tid)
try:
return run_task(task, tid)
except KeyboardInterrupt:
sys.exit(0)
def run_task_in_process(task, tid, environ):
from env import env
env.start(False, environ)
logger.debug("run task in process %s %s", task, tid)
try:
return run_task(task, tid)
except KeyboardInterrupt:
sys.exit(0)
def start(self):
if self.started:
return
env.start(True, isLocal=self.isLocal)
self.scheduler.start()
self.started = True
atexit.register(self.stop)
def handler(signm, frame):
logger.error("got signal %d, exit now", signm)
self.scheduler.shutdown()
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
def start(self):
if self.started:
return
env.start(True, isLocal=self.isLocal)
self.scheduler.start()
self.started = True
atexit.register(self.stop)
def handler(signm, frame):
logger.error("got signal %d, exit now", signm)
self.scheduler.shutdown()
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
