def test_job_conf(self):
job_conf = {}
for k in mrv1_to_mrv2:
job_conf[k] = k
jc = JobConf(
[item for sublist in iteritems(job_conf) for item in sublist])
for k in mrv2_to_mrv1:
self.assertEqual(jc[k], job_conf[mrv2_to_mrv1[k]])
def dump_counters(hs, logger):
counters = hs.get_counters()
for phase in ['mapping', 'reducing']:
logger.info("%s counters:", phase.capitalize())
for group in counters[phase]:
logger.info(" Group %s", group)
for c, v in iteritems(counters[phase][group]):
logger.info(" %s: %s", c, v)
def dump_counters(hs, logger):
counters = hs.get_counters()
for phase in ['mapping', 'reducing']:
logger.info("%s counters:", phase.capitalize())
for group in counters[phase]:
logger.info(" Group %s", group)
for c, v in iteritems(counters[phase][group]):
logger.info(" %s: %s", c, v)
def test_job_conf(self):
job_conf = {}
for k in mrv1_to_mrv2:
job_conf[k] = k
jc = JobConf(
[item for sublist in iteritems(job_conf) for item in sublist]
)
for k in mrv2_to_mrv1:
self.assertEqual(jc[k], job_conf[mrv2_to_mrv1[k]])
def __serialize_as_needed(self, key, value):
out_kv = {'K': key, 'V': value}
jc = self.job_conf
if AVRO_OUTPUT in jc and (self.is_reducer() or self.__is_map_only()):
for mode, record in iteritems(out_kv):
serializer = self.__serializers.get(mode)
if serializer is not None:
out_kv[mode] = serializer.serialize(record)
return out_kv['K'], out_kv['V']
def check_counts(self, fname, exp_count):
count = count_outputs(fname)
try:
for k, v in iteritems(exp_count):
self.assertTrue(k in count)
self.assertEqual(count[k], v)
except AssertionError:
print(count)
raise
def run(self):
wc = {}
if os.path.isdir(self.input_path):
for fn in os.listdir(self.input_path):
if fn[0] == ".":
continue
self._wordcount_file(wc, fn, self.input_path)
else:
self._wordcount_file(wc, self.input_path)
if self.min_occurrence:
wc = dict(t for t in iteritems(wc) if t[1] >= self.min_occurrence)
return wc
def run(self):
wc = {}
if os.path.isdir(self.input_path):
for fn in os.listdir(self.input_path):
if fn[0] == ".":
continue
self._wordcount_file(wc, fn, self.input_path)
else:
self._wordcount_file(wc, self.input_path)
if self.min_occurrence:
wc = dict(t for t in iteritems(wc) if t[1] >= self.min_occurrence)
return wc
def __run_test(self, mode, mapper_class, context_class):
cmd_file = self.__write_cmd_file(mode)
pp.run_task(
pp.Factory(mapper_class=mapper_class), private_encoding=False,
context_class=context_class, cmd_file=cmd_file)
out_fn = cmd_file + '.out'
out_records = []
with open(out_fn, 'rb') as f:
bf = BinaryDownStreamAdapter(f)
for cmd, args in bf:
if cmd == bf.OUTPUT:
name, color = args
out_records.append({'name': name, 'favorite_color': color})
self.assertEqual(len(out_records), len(self.records))
for out_r, r in zip(out_records, self.records):
for k, v in iteritems(out_r):
self.assertEqual(v.decode('UTF-8'), r[k])
def spill_all(self):
self.ctx.increment_counter(self.spill_counter, 1)
self.ctx.increment_counter(self.spilled_bytes_counter, self.used_bytes)
ctx = self.ctx
writer = ctx.writer
ctx.writer = None
# disable auto-deserialize (ctx.key will be called by reduce)
# FIXME: this might break custom Context implementations
get_input_key = ctx.get_input_key
ctx.get_input_key = types.MethodType(lambda self: self._key, ctx)
for key, values in iteritems(self.data):
ctx._key, ctx._values = key, iter(values)
self.reducer.reduce(ctx)
ctx.writer = writer
ctx.get_input_key = get_input_key
self.data.clear()
self.used_bytes = 0
def __run_test(self, mode, mapper_class, context_class):
cmd_file = self.__write_cmd_file(mode)
pp.run_task(pp.Factory(mapper_class=mapper_class),
private_encoding=False,
context_class=context_class,
cmd_file=cmd_file)
out_fn = cmd_file + '.out'
out_records = []
with open(out_fn, 'rb') as f:
bf = BinaryDownStreamAdapter(f)
for cmd, args in bf:
if cmd == bf.OUTPUT:
name, color = args
out_records.append({'name': name, 'favorite_color': color})
self.assertEqual(len(out_records), len(self.records))
for out_r, r in zip(out_records, self.records):
for k, v in iteritems(out_r):
self.assertEqual(v.decode('UTF-8'), r[k])
def main(exp, res):
expected = {}
for l in iter_lines(exp):
p = l.strip().split(';')
expected.setdefault(p[1], Counter())[p[2]] += 1
computed = {}
for l in iter_lines(res):
p = l.strip().split('\t')
computed[p[0]] = eval(p[1])
if set(computed) != set(expected):
sys.exit("ERROR: computed keys != expected keys: %r != %r" %
(sorted(computed), sorted(expected)))
for k, v in iteritems(expected):
if computed[k] != v:
sys.exit("ERROR: %r: %r != %r" % (k, computed[k], dict(v)))
print('All is ok!')
def main(exp, res):
expected = {}
for l in iter_lines(exp):
p = l.strip().split(';')
expected.setdefault(p[1], Counter())[p[2]] += 1
computed = {}
for l in iter_lines(res):
p = l.strip().split('\t')
computed[p[0]] = eval(p[1])
if set(computed) != set(expected):
sys.exit("ERROR: computed keys != expected keys: %r != %r" % (
sorted(computed), sorted(expected)))
for k, v in iteritems(expected):
if computed[k] != v:
sys.exit("ERROR: %r: %r != %r" % (k, computed[k], dict(v)))
print('All is ok!')
def get_counters(self):
r"""
Extract counters information accumulated by this simulator instance.
The expected usage is as follows::
.. code-block:: python
counters = hs.get_counters()
for phase in ['mapping', 'reducing']:
print "{} counters:".format(phase.capitalize())
for group in counters[phase]:
print " Group {}".format(group)
for c, v in counters[phase][group].iteritems():
print " {}: {}".format(c, v)
"""
ctable = {'mapping': {}, 'reducing': {}}
for k, v in iteritems(self.counters):
ctable.setdefault(
k[0], {}).setdefault(v[0][0], {}).setdefault(v[0][1], v[1])
return ctable
def write_header_down_stream(self, down_stream, authorization, job_conf):
out_jc = sum([[k, v] for k, v in iteritems(job_conf)], [])
self.write_authorization(down_stream, authorization)
down_stream.send(down_stream.START_MESSAGE, 0)
down_stream.send(down_stream.SET_JOB_CONF, *out_jc)