def __init__(self, cfg, trace_every=0,
get_body=None, get_label=None, get_prefix=None, seed=None,
min_support=None, normalizer=None, tokenizer=None):
"""Read configuration"""
self.cfg = cfg
self._get_body = get_body
self._get_label = get_label
self._get_prefix = get_prefix
self.trace_every = trace_every
# Set options
self.random_state = cfg['random_state'] if seed is None else seed
cfg_signer = cfg['signer']
self.min_support = cfg_signer['min_support'] if min_support is None else min_support
# tokenizer
self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer
# Configure minhash signer
sig_width = cfg_signer['width']
lsh_hasher = LSHC(width=sig_width, seed=self.random_state, **cfg_signer['lsh'])
self.signer = MinHashSignature(sig_width,
lsh_hasher=lsh_hasher,
universe_size=cfg_signer['universe_size'],
kmin=cfg_signer['kmin'],
seed=self.random_state)
# Configure shingler
cfg_key_shingle = cfg['shingler']
self.shingler = get_default_shingler(**cfg_key_shingle)
# Configure sketch comparison algorithm
self.sketch_enabled = False
self.cluster_builder = Cluster(min_support=self.min_support)
def __init__(self, cfg, trace_every=0,
content_field='content',
get_body=None, get_label=None, get_prefix=None, min_support=None,
seed=0, tokenizer=None):
"""Read configuration"""
self.cfg = cfg
self._get_body = get_body
self._get_label = get_label
self._get_prefix = get_prefix
self.trace_every = trace_every
self.get_content = operator.itemgetter(content_field)
# Set options
self.min_support = cfg['min_support'] if min_support is None else min_support
# Tokenizer
self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer
# Configure minhash signer
sig_width = cfg['sig_width']
lsh_hasher = LSHC(width=sig_width, **cfg['lsh_options'])
self.signer = MinHashSignature(sig_width,
lsh_hasher=lsh_hasher,
kmin=cfg['kmin'])
# Configure shingler
cfg_key_shingle = cfg['shingler']
self.shingler = get_default_shingler(**cfg_key_shingle)
# Configure sketch comparison algorithm
cfg_sketch = cfg['sketch']
self.sketch_enabled = cfg_sketch['enabled']
self.sketch_dist_fn = None
self.max_dist = None
if self.sketch_enabled:
algorithm_name = cfg_sketch['algorithm']
try:
sketch_algorithm = getattr(SketchModel, algorithm_name)
except AttributeError:
raise RuntimeError("Unknown sketch model specified: '%s'"
% algorithm_name)
self.sketch_bits = cfg_sketch['size']
cfg_sketch_shingler = cfg_sketch['shingler']
if not cfg_sketch_shingler['enabled']:
# if sketch shingler is disabled, we also disable signer
# as we will use default signer
self.sketch_shingler = None
self.sketch_signer = None
elif sketch_algorithm == SketchModel.simhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = SimHashSignature(self.sketch_bits, seed=seed)
elif sketch_algorithm == SketchModel.minhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = MinHashSketchSignature(self.sketch_bits, seed=seed)
self.sketch_shingler._tokenizer = None
self.max_dist = \
int(floor(self.sketch_bits *
(1.0 - float(cfg_sketch['resemblance']))))
self.sketch_dist_fn = hamming
self.sketch_operator = OPERATOR_MAP[cfg_sketch.get('operator', 'and')]
self.cluster_builder = Cluster(sketch_dist_fn=self.sketch_dist_fn,
max_dist=self.max_dist,
min_support=self.min_support,
sketch_operator=self.sketch_operator)
class HDClustering(object):
def __init__(self, cfg, trace_every=0,
get_body=None, get_label=None, get_prefix=None, seed=None,
min_support=None, normalizer=None, tokenizer=None):
"""Read configuration"""
self.cfg = cfg
self._get_body = get_body
self._get_label = get_label
self._get_prefix = get_prefix
self.trace_every = trace_every
# Set options
self.random_state = cfg['random_state'] if seed is None else seed
cfg_signer = cfg['signer']
self.min_support = cfg_signer['min_support'] if min_support is None else min_support
# tokenizer
self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer
# Configure minhash signer
sig_width = cfg_signer['width']
lsh_hasher = LSHC(width=sig_width, seed=self.random_state, **cfg_signer['lsh'])
self.signer = MinHashSignature(sig_width,
lsh_hasher=lsh_hasher,
universe_size=cfg_signer['universe_size'],
kmin=cfg_signer['kmin'],
seed=self.random_state)
# Configure shingler
cfg_key_shingle = cfg['shingler']
self.shingler = get_default_shingler(**cfg_key_shingle)
# Configure sketch comparison algorithm
self.sketch_enabled = False
self.cluster_builder = Cluster(min_support=self.min_support)
def _map_iter(self, data):
"""Find clusters in an iterable"""
get_body = self._get_body
get_label = self._get_label
get_prefix = self._get_prefix
for i, obj in enumerate(data):
body = obj if get_body is None else get_body(obj)
label = i if get_label is None else get_label(obj)
prefix = None if get_prefix is None else get_prefix(obj)
for feat in self._map_item(obj, body, label, prefix):
yield feat
def _map_item(self, obj, body, label, prefix=None):
# Extract features
obj_content = obj['content']
normalized_content, meta = self.normalizer.normalize(obj_content)
content_tokens = self.tokenizer.tokenize(normalized_content)
features = self.shingler.get_shingles(content_tokens, prefix=prefix)
keys = self.signer.get_signature(features)
sketch = None
yield (keys, (label, sketch))
def clusters_from_iter(self, data):
"""Find clusters in an iterable"""
cluster_builder = self.cluster_builder
trace_every = self.trace_every
for i, obj in enumerate(self._map_iter(data)):
if trace_every > 0 and (not i % trace_every):
LOG.info("Processing line " + str(i))
keys, val = obj
label, sketch = val \
if isinstance(val, tuple) \
else (val, None)
cluster_builder.add_item(keys, label=label, sketch=sketch)
return cluster_builder.get_clusters()
def mapper(self, obj):
"""Perform a mapper task in MR"""
get_body = self._get_body
get_label = self._get_label
get_prefix = self._get_prefix
body = obj if get_body is None else get_body(obj)
label = obj if get_label is None else get_label(obj)
prefix = None if get_prefix is None else get_prefix(obj)
for keys, val in self._map_item(obj, body, label, prefix):
for key in keys:
yield key, val
def reducer(self, key, tuple_gen):
"""Perform a reducer task in MR
If sketches enabled, data consists of::
(key, [(lbl, sk), (lbl, sk), (lbl, sk)])
Otherwise::
(key, [lbl, lbl, lbl])
"""
# If not using sketches, we are done
return key, list(set(tuple_gen))
class HDClustering(object):
def __init__(self, cfg, trace_every=0,
content_field='content',
get_body=None, get_label=None, get_prefix=None, min_support=None,
seed=0, tokenizer=None):
"""Read configuration"""
self.cfg = cfg
self._get_body = get_body
self._get_label = get_label
self._get_prefix = get_prefix
self.trace_every = trace_every
self.get_content = operator.itemgetter(content_field)
# Set options
self.min_support = cfg['min_support'] if min_support is None else min_support
# Tokenizer
self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer
# Configure minhash signer
sig_width = cfg['sig_width']
lsh_hasher = LSHC(width=sig_width, **cfg['lsh_options'])
self.signer = MinHashSignature(sig_width,
lsh_hasher=lsh_hasher,
kmin=cfg['kmin'])
# Configure shingler
cfg_key_shingle = cfg['shingler']
self.shingler = get_default_shingler(**cfg_key_shingle)
# Configure sketch comparison algorithm
cfg_sketch = cfg['sketch']
self.sketch_enabled = cfg_sketch['enabled']
self.sketch_dist_fn = None
self.max_dist = None
if self.sketch_enabled:
algorithm_name = cfg_sketch['algorithm']
try:
sketch_algorithm = getattr(SketchModel, algorithm_name)
except AttributeError:
raise RuntimeError("Unknown sketch model specified: '%s'"
% algorithm_name)
self.sketch_bits = cfg_sketch['size']
cfg_sketch_shingler = cfg_sketch['shingler']
if not cfg_sketch_shingler['enabled']:
# if sketch shingler is disabled, we also disable signer
# as we will use default signer
self.sketch_shingler = None
self.sketch_signer = None
elif sketch_algorithm == SketchModel.simhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = SimHashSignature(self.sketch_bits, seed=seed)
elif sketch_algorithm == SketchModel.minhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = MinHashSketchSignature(self.sketch_bits, seed=seed)
self.sketch_shingler._tokenizer = None
self.max_dist = \
int(floor(self.sketch_bits *
(1.0 - float(cfg_sketch['resemblance']))))
self.sketch_dist_fn = hamming
self.sketch_operator = OPERATOR_MAP[cfg_sketch.get('operator', 'and')]
self.cluster_builder = Cluster(sketch_dist_fn=self.sketch_dist_fn,
max_dist=self.max_dist,
min_support=self.min_support,
sketch_operator=self.sketch_operator)
def _map_iter(self, data):
"""Find clusters in an iterable"""
get_body = self._get_body
get_label = self._get_label
get_prefix = self._get_prefix
for i, obj in enumerate(data):
body = obj if get_body is None else get_body(obj)
label = i if get_label is None else get_label(obj)
prefix = None if get_prefix is None else get_prefix(obj)
for feat in self._map_item(obj, body, label, prefix):
yield feat
def _map_item(self, obj, body, label, prefix=None):
# Extract features
obj_content = self.get_content(obj)
content_tokens = self.tokenizer.tokenize(obj_content)
features = self.shingler.get_shingles(content_tokens, prefix=prefix)
if self.sketch_enabled and (self.sketch_shingler is None or self.sketch_signer is None):
keys, sketch = self.signer.get_signature(features, with_sketch=True)
elif self.sketch_enabled and (self.sketch_shingler is not None and self.sketch_signer is not None):
keys = self.signer.get_signature(features)
sketch_features = self.sketch_shingler.get_shingles(content_tokens)
sketch = self.sketch_signer.get_signature(sketch_features)
else:
keys = self.signer.get_signature(features)
sketch = None
yield (keys, (label, sketch))
def clusters_from_iter(self, data):
"""Find clusters in an iterable"""
cluster_builder = self.cluster_builder
trace_every = self.trace_every
for i, obj in enumerate(self._map_iter(data)):
if trace_every > 0 and (not i % trace_every):
LOG.info("Processing line " + str(i))
keys, val = obj
label, sketch = val \
if isinstance(val, tuple) \
else (val, None)
cluster_builder.add_item(keys, label=label, sketch=sketch)
return cluster_builder.get_clusters()
def mapper(self, obj):
"""Perform a mapper task in MR"""
get_body = self._get_body
get_label = self._get_label
get_prefix = self._get_prefix
body = obj if get_body is None else get_body(obj)
label = obj if get_label is None else get_label(obj)
prefix = None if get_prefix is None else get_prefix(obj)
for keys, val in self._map_item(obj, body, label, prefix):
for key in keys:
yield key, val
def reducer(self, key, tuple_gen):
"""Perform a reducer task in MR
If sketches enabled, data consists of::
(key, [(lbl, sk), (lbl, sk), (lbl, sk)])
Otherwise::
(key, [lbl, lbl, lbl])
"""
# If not using sketches, we are done
if self.sketch_dist_fn is None:
return key, list(set(tuple_gen))
# create a dict mappipng a label to a sketch
return key, dict(tuple_gen).items()