def runTest(self):
host, port = self.test_server_address
input = chainify(['http://{0}/{1}:{2}'.format(node, host, port)] * N
for node in self.blacklist)
self.job = BlacklistJob().run(input=input)
self.assertAllEqual(sorted(k for k, v in self.results(self.job)),
chainify([node] * N for node in self.blacklist))
def prepare_input_map(self, task, stage, params):
# The input map maps a label to a sequence of inputs with that
# label.
map = defaultdict(list)
for l, i in util.chainify(self.labelexpand(task, stage, i, params) for i in self.get_inputs()):
if stage.combine_labels:
map[0].append(i)
else:
map[l].append(i)
if stage.sort:
newmap = {}
if stage.input_sorted:
for label, inputs in map.iteritems():
input = merge(*(merge_wrapper(inp, sort_range=stage.sort, desc=stage.desc) for inp in inputs))
newmap[label] = [input]
else:
for label, inputs in map.iteritems():
input = chainify(shuffled(inputs))
newmap[label] = [disk_sort(input,
task.path('sort.dl'),
sort_keys=stage.sort,
sort_buffer_size='15%',
binaries=stage.binaries,
desc=stage.desc)]
map = newmap
#print "OUTSIE: %s" % str(map)
return map
def runTest(self):
host, port = self.test_server_address
input = chainify(['http://{0}/{1}:{2}'.format(node, host, port)] * N
for node in self.blacklist)
self.job = BlacklistJob().run(input=input)
self.assertAllEqual(sorted(k for k, v in self.results(self.job)),
chainify([node] * N for node in self.blacklist))
def prepare_input_map(self, task, stage, params):
# The input map maps a label to a sequence of inputs with that
# label.
map = defaultdict(list)
for l, i in util.chainify(self.labelexpand(task, stage, i, params)
for i in self.get_inputs()):
if stage.combine_labels:
map[0].append(i)
else:
map[l].append(i)
if stage.sort:
newmap = {}
if stage.input_sorted:
for label, inputs in map.iteritems():
input = merge(*(merge_wrapper(inp,
sort_range=stage.sort,
desc=stage.desc)
for inp in inputs))
newmap[label] = [input]
else:
for label, inputs in map.iteritems():
input = chainify(shuffled(inputs))
newmap[label] = [disk_sort(input,
task.path('sort.dl'),
sort_keys=stage.sort,
sort_buffer_size='15%',
binaries=stage.binaries,
desc=stage.desc)]
map = newmap
#print "OUTSIE: %s" % str(map)
return map
def runTest(self):
# assumption: scheduler starts scheduling tasks in the order of input
host, port = self.test_server_address
input = chainify(['http://{0}/{1}:{2}'.format(node, host, port)] * N
for node in self.blacklist)
self.job = BlacklistJob().run(input=input)
self.assertAllEqual(sorted(k for k, v in self.results(self.job)),
chainify([node] * N for node in self.blacklist))
def runTest(self):
# assumption: scheduler starts scheduling tasks in the order of input
host, port = self.test_server_address
input = chainify(['http://{0}/{1}:{2}'.format(node, host, port)] * N
for node in self.blacklist)
self.job = BlacklistJob().run(input=input)
self.assertAllEqual(sorted(k for k, v in self.results(self.job)),
chainify([node] * N for node in self.blacklist))
def prepare_input_map(self, task, stage, params):
# The input map maps a label to a sequence of inputs with that
# label.
map = defaultdict(list)
for l, i in util.chainify(self.labelexpand(task, stage, i, params) for i in self.get_inputs()):
map[l].append(i)
return map
def prepare_input_map(self, task, stage, params):
# The input map maps a label to a sequence of inputs with that
# label.
map = defaultdict(list)
for l, i in util.chainify(self.labelexpand(task, stage, i, params)
for i in self.get_inputs()):
map[l].append(i)
return map
def walk(self):
for job in self:
status, results = job.results()
if status == 'unknown job':
input = util.chainify(self.inputs(job))
if not any(i is None for i in input):
job.run(input=input)
elif status == 'active':
pass
elif status == 'ready':
yield 1
else:
raise JobError(job, "Status %s" % status)
def walk(self):
for job in self:
status, results = job.results()
if status == 'unknown job':
input = util.chainify(self.inputs(job))
if not any(i is None for i in input):
job.run(input=input)
elif status == 'active':
pass
elif status == 'ready':
yield 1
else:
raise JobError(job, "Status %s" % status)
