def init():
global HTTP_PORT, PARAMS_FILE, EXT_MAP, EXT_REDUCE,\
PART_SUFFIX, MAP_OUTPUT, REDUCE_DL,\
REDUCE_SORTED, REDUCE_OUTPUT, OOB_FILE,\
JOB_HOME, DISCO_ROOT, JOB_ROOT, PART_OUTPUT,\
MAP_INDEX, REDUCE_INDEX, REQ_FILES, CHDIR_PATH
tmp, HTTP_PORT, DISCO_ROOT = load_conf()
job_name = this_name()
JOB_HOME = "%s/%s/%s/" %\
(this_host(), md5.md5(job_name).hexdigest()[:2], job_name)
JOB_ROOT = "%s/data/%s" % (DISCO_ROOT, JOB_HOME)
if resultfs_enabled:
pp = "%s/temp/%s" % (DISCO_ROOT, JOB_HOME)
else:
pp = JOB_ROOT
CHDIR_PATH = pp
PARAMS_FILE = pp + "params.dl"
REQ_FILES = pp + "lib"
EXT_MAP = pp + "ext.map"
EXT_REDUCE = pp + "ext.reduce"
PART_SUFFIX = "-%.9d"
MAP_OUTPUT = pp + "map-disco-%d" + PART_SUFFIX
PART_OUTPUT = pp + "part-disco-%.9d"
REDUCE_DL = pp + "reduce-in-%d.dl"
REDUCE_SORTED = pp + "reduce-in-%d.sorted"
REDUCE_OUTPUT = pp + "reduce-disco-%d"
OOB_FILE = pp + "oob/%s"
MAP_INDEX = pp + "map-index.txt"
REDUCE_INDEX = pp + "reduce-index.txt"
def result_iterator(results, notifier = None,\
proxy = None, reader = func.netstr_reader):
res = []
for dir_url in results:
if dir_url.startswith("dir://"):
res += util.parse_dir(dir_url, proxy)
else:
res.append(dir_url)
x, x, root = util.load_conf()
for url in res:
if url.startswith("file://"):
fname = url[7:]
fd = file(fname)
sze = os.stat(fname).st_size
elif url.startswith("disco://"):
host, fname = url[8:].split("/", 1)
url = util.proxy_url(proxy, fname, host)
if util.resultfs_enabled:
f = "%s/data/%s" % (root, fname)
fd = file(f)
sze = os.stat(f).st_size
else:
sze, fd = comm.open_remote(url)
else:
raise JobException("Invalid result url: %s" % url)
if notifier:
notifier(url)
for x in reader(fd, sze, fname):
yield x
def init():
global HTTP_PORT, LOCAL_PATH, PARAMS_FILE, EXT_MAP, EXT_REDUCE,\
MAP_OUTPUT, CHUNK_OUTPUT, REDUCE_DL, REDUCE_SORTED, REDUCE_OUTPUT
tmp, HTTP_PORT, LOCAL_PATH = load_conf()
PARAMS_FILE = LOCAL_PATH + "%s/params"
EXT_MAP = LOCAL_PATH + "%s/ext-map"
EXT_REDUCE = LOCAL_PATH + "%s/ext-reduce"
MAP_OUTPUT = LOCAL_PATH + "%s/map-disco-%d-%.9d"
CHUNK_OUTPUT = LOCAL_PATH + "%s/map-chunk-%d"
REDUCE_DL = LOCAL_PATH + "%s/reduce-in-%d.dl"
REDUCE_SORTED = LOCAL_PATH + "%s/reduce-in-%d.sorted"
REDUCE_OUTPUT = LOCAL_PATH + "%s/reduce-disco-%d"
def init():
global HTTP_PORT, LOCAL_PATH, PARAMS_FILE, EXT_MAP, EXT_REDUCE,\
PART_SUFFIX, MAP_OUTPUT, CHUNK_OUTPUT, REDUCE_DL,\
REDUCE_SORTED, REDUCE_OUTPUT, OOB_FILE, OOB_URL
tmp, HTTP_PORT, LOCAL_PATH = load_conf()
OOB_URL = ("http://%s/disco/ctrl/oob_get?" % this_master())\
+ "name=%s&key=%s"
PARAMS_FILE = LOCAL_PATH + "%s/params"
EXT_MAP = LOCAL_PATH + "%s/ext-map"
EXT_REDUCE = LOCAL_PATH + "%s/ext-reduce"
PART_SUFFIX = "-%.9d"
MAP_OUTPUT = LOCAL_PATH + "%s/map-disco-%d" + PART_SUFFIX
CHUNK_OUTPUT = LOCAL_PATH + "%s/map-chunk-%d"
REDUCE_DL = LOCAL_PATH + "%s/reduce-in-%d.dl"
REDUCE_SORTED = LOCAL_PATH + "%s/reduce-in-%d.sorted"
REDUCE_OUTPUT = LOCAL_PATH + "%s/reduce-disco-%d"
OOB_FILE = LOCAL_PATH + "%s/oob/%s"