def test_host_names():
urls = StringCounter()
urls['http://www.example.com/folder1'] = 3
urls['http://www.example.com/folder2'] = 2
urls['http://www.different.com/folder2'] = 7
assert features.host_names(urls) == StringCounter({
'www.example.com': 5,
'www.different.com': 7,
})
def test_host_names():
urls = StringCounter()
urls['http://www.example.com/folder1'] = 3
urls['http://www.example.com/folder2'] = 2
urls['http://www.different.com/folder2'] = 7
assert features.host_names(urls) == StringCounter({
'www.example.com': 5,
'www.different.com': 7,
})
def html_to_fc(html=None,
clean_html=None,
clean_visible=None,
encoding=None,
url=None,
timestamp=None,
other_features=None):
'''`html` is expected to be a raw string received over the wire from a
remote webserver, and `encoding`, if provided, is used to decode
it. Typically, encoding comes from the Content-Type header field.
The :func:`~streamcorpus_pipeline._clean_html.make_clean_html`
function handles character encodings.
'''
def add_feature(name, xs):
if name not in fc:
fc[name] = StringCounter()
fc[name] += StringCounter(xs)
timestamp = timestamp or int(time.time() * 1000)
other_features = other_features or {}
if clean_html is None:
if html is not None:
try:
clean_html_utf8 = make_clean_html(html, encoding=encoding)
except:
logger.warn('dropping doc because:', exc_info=True)
return
clean_html = clean_html_utf8.decode('utf-8')
else:
clean_html_utf8 = u''
clean_html = u''
else:
clean_html_utf8 = u''
if clean_visible is None or len(clean_visible) == 0:
clean_visible = make_clean_visible(clean_html_utf8).decode('utf-8')
elif isinstance(clean_visible, str):
clean_visible = clean_visible.decode('utf-8')
fc = FeatureCollection()
fc[u'meta_raw'] = html and uni(html, encoding) or u''
fc[u'meta_clean_html'] = clean_html
fc[u'meta_clean_visible'] = clean_visible
fc[u'meta_timestamp'] = unicode(timestamp)
url = url or u''
fc[u'meta_url'] = uni(url)
add_feature(u'icq', features.ICQs(clean_visible))
add_feature(u'skype', features.skypes(clean_visible))
add_feature(u'phone', features.phones(clean_visible))
add_feature(u'email', features.emails(clean_visible))
bowNP, normalizations = features.noun_phrases(cleanse(clean_visible),
included_unnormalized=True)
add_feature(u'bowNP', bowNP)
bowNP_unnorm = chain(*normalizations.values())
add_feature(u'bowNP_unnorm', bowNP_unnorm)
add_feature(u'image_url', features.image_urls(clean_html))
add_feature(u'a_url', features.a_urls(clean_html))
## get parsed versions, extract usernames
fc[u'img_url_path_dirs'] = features.path_dirs(fc[u'image_url'])
fc[u'img_url_hostnames'] = features.host_names(fc[u'image_url'])
fc[u'usernames'] = features.usernames(fc[u'image_url'])
fc[u'a_url_path_dirs'] = features.path_dirs(fc[u'a_url'])
fc[u'a_url_hostnames'] = features.host_names(fc[u'a_url'])
fc[u'usernames'] += features.usernames(fc[u'a_url'])
#fc[u'usernames'] += features.usernames2(
# fc[u'meta_clean_visible'])
# beginning of treating this as a pipeline...
xform = features.entity_names()
fc = xform.process(fc)
for feat_name, feat_val in other_features.iteritems():
fc[feat_name] += StringCounter(feat_val)
return fc
def html_to_fc(html=None, clean_html=None, clean_visible=None, encoding=None, url=None,
timestamp=None, other_features=None):
'''`html` is expected to be a raw string received over the wire from a
remote webserver, and `encoding`, if provided, is used to decode
it. Typically, encoding comes from the Content-Type header field.
The :func:`~streamcorpus_pipeline._clean_html.make_clean_html`
function handles character encodings.
'''
def add_feature(name, xs):
if name not in fc:
fc[name] = StringCounter()
fc[name] += StringCounter(xs)
timestamp = timestamp or int(time.time() * 1000)
other_features = other_features or {}
if clean_html is None:
if html is not None:
try:
clean_html_utf8 = make_clean_html(html, encoding=encoding)
except:
logger.warn('dropping doc because:', exc_info=True)
return
clean_html = clean_html_utf8.decode('utf-8')
else:
clean_html_utf8 = u''
clean_html = u''
else:
clean_html_utf8 = u''
if clean_visible is None or len(clean_visible) == 0:
clean_visible = make_clean_visible(clean_html_utf8).decode('utf-8')
elif isinstance(clean_visible, str):
clean_visible = clean_visible.decode('utf-8')
fc = FeatureCollection()
fc[u'meta_raw'] = html and uni(html, encoding) or u''
fc[u'meta_clean_html'] = clean_html
fc[u'meta_clean_visible'] = clean_visible
fc[u'meta_timestamp'] = unicode(timestamp)
url = url or u''
fc[u'meta_url'] = uni(url)
add_feature(u'icq', features.ICQs(clean_visible))
add_feature(u'skype', features.skypes(clean_visible))
add_feature(u'phone', features.phones(clean_visible))
add_feature(u'email', features.emails(clean_visible))
bowNP, normalizations = features.noun_phrases(
cleanse(clean_visible), included_unnormalized=True)
add_feature(u'bowNP', bowNP)
bowNP_unnorm = chain(*normalizations.values())
add_feature(u'bowNP_unnorm', bowNP_unnorm)
add_feature(u'image_url', features.image_urls(clean_html))
add_feature(u'a_url', features.a_urls(clean_html))
## get parsed versions, extract usernames
fc[u'img_url_path_dirs'] = features.path_dirs(fc[u'image_url'])
fc[u'img_url_hostnames'] = features.host_names(fc[u'image_url'])
fc[u'usernames'] = features.usernames(fc[u'image_url'])
fc[u'a_url_path_dirs'] = features.path_dirs(fc[u'a_url'])
fc[u'a_url_hostnames'] = features.host_names(fc[u'a_url'])
fc[u'usernames'] += features.usernames(fc[u'a_url'])
#fc[u'usernames'] += features.usernames2(
# fc[u'meta_clean_visible'])
# beginning of treating this as a pipeline...
xform = features.entity_names()
fc = xform.process(fc)
for feat_name, feat_val in other_features.iteritems():
fc[feat_name] += StringCounter(feat_val)
return fc