def TestInput(data):
fdp = atheris.FuzzedDataProvider(data)
try:
ftfy.fix_text(fdp.ConsumeString(1000))
ftfy.fix_text(fdp.ConsumeUnicode(1000))
plan1 = ftfy.fix_and_explain(fdp.ConsumeString(1000))[1]
plan2 = ftfy.fix_and_explain(fdp.ConsumeUnicode(1000))[1]
ftfy.apply_plan(fdp.ConsumeString(1000), plan1)
ftfy.apply_plan(fdp.ConsumeString(1000), plan2)
ftfy.apply_plan(fdp.ConsumeUnicode(1000), plan1)
ftfy.apply_plan(fdp.ConsumeUnicode(1000), plan2)
ftfy.fix_text_segment(fdp.ConsumeString(1000))
ftfy.fix_text_segment(fdp.ConsumeUnicode(1000))
f = open("temp.txt", "w")
f.write(fdp.ConsumeString(1000))
f.write(fdp.ConsumeUnicode(1000))
f.close()
f = open("temp.txt", "r")
ftfy.fix_file(f)
f.close()
ftfy.guess_bytes(fdp.ConsumeBytes(1000))
except UnicodeError as e:
if "Hey wait, this isn't Unicode." not in str(e):
raise e
def clean(datapath):
"""
Fix encoding errors in a data file and
gets rid of data which still seems problematic.
"""
red_flags = ['â€', 'Â']
with open(datapath, 'r') as file:
data = json.load(file)
bad = []
good = []
for article in progress(data, 'Fixing {0} articles...'.format(len(data))):
for key in ['title', 'text']:
article[key] = fix_text_segment(article[key])
flagged = False
for flag in red_flags:
if flag in article['text'] + article['title']:
bad.append(article)
flagged = True
break
if not flagged:
good.append(article)
print('Getting rid of {0} bad articles.'.format(len(bad)))
outpath = datapath.replace('.json', '_cleaned.json')
with open(outpath, 'w') as file:
json.dump(good, file)
def format(self, data):
super().format(data)
rn = self._xml.find('.//items/.')
if len(data):
for item in data:
items_rt = ET.XML('<item></item>')
for el in item:
items_rt.append(el)
rn.append(items_rt)
for el in rn.iter():
if el.text:
el.text = ftfy.fix_text_segment(el.text, fix_entities='auto', remove_terminal_escapes=True, fix_encoding=True, fix_latin_ligatures=True, fix_character_width=True, uncurl_quotes=True, fix_line_breaks=True, fix_surrogates=True, remove_control_chars=True, remove_bom=True, normalization='NFKC')
# remove non breaking space code
for p in self._xml.itertext():
re.sub(r'ğŸ', '', p)
try:
p = self._xml.findall('.//headers/..')
for e in p:
e.remove(e.find('./headers'))
except AttributeError:
pass
try:
p = self._xml.findall('.//paging/..')
for e in p:
e.remove(e.find('./paging'))
except AttributeError:
pass
if self.__class__.__name__ is 'SaveFaceFormatterXML':
if self._formatter_func is not None:
self._xml = self._formatter_func(self._xml)
def test_entities():
example = '&\n<html>\n&'
eq_(fix_text(example), '&\n<html>\n&')
eq_(fix_text_segment(example), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment('<>', fix_entities=False), '<>')
eq_(fix_text_segment('<>', fix_entities=True), '<>')
eq_(fix_text_segment('<>'), '<>')
eq_(fix_text_segment('jednocześnie'), 'jednocześnie')
eq_(fix_text_segment('JEDNOCZEŚNIE'), 'JEDNOCZEŚNIE')
def load_metadata(soup):
if not soup:
return {}
metas = soup.select("script[type='application/ld+json']")
o = {"errors": []}
for meta in metas:
try:
_o = json.loads(meta.string)
except Exception as e:
o["errors"].append([f"{e.__class__.__name__} :: {e}", meta.string])
continue
if isinstance(_o, list):
for __o in _o:
o[__o["@type"]] = __o
elif "@type" in _o and isinstance(_o["@type"], list):
for t in _o["@type"]:
for __o in _o:
o[t] = __o
elif "@type" in _o and _o["@type"] in o:
o[_o["@type"]] = _o
else:
continue
flat = FlatterDict(o)
o = {}
for k, v in flat.items():
if (isinstance(v, str) and not re.search(r"(image|url)", k)
and not re.match(r"\s*http", v)):
txt = fix_text_segment(v)
if txt and "<" in txt:
txt = re.sub(r"<[^>]*>", "", txt)
o[k] = txt
else:
o[k] = v
flat = json.loads(
json.dumps(
{k: v
for k, v in o.items()},
indent=4,
default=lambda x: dict(**x) if isinstance(x, FlatterDict) else x,
))
print(flat)
return flat
def test_entities():
example = '&\n<html>\n&'
eq_(fix_text(example), '&\n<html>\n&')
eq_(fix_text_segment(example), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment('<>', fix_entities=False), '<>')
eq_(fix_text_segment('<>', fix_entities=True), '<>')
eq_(fix_text_segment('<>'), '<>')
def __init__(self,
url,
html,
row,
soup=None,
lxml=None,
fix_encoding_errors=True):
self.url = url
self.sitemap_data = row
self.html = (fix_text_segment(html.replace("\xa0", " "),
uncurl_quotes=False)
if fix_encoding_errors else html)
try:
self.soup = soup if soup else BeautifulSoup(self.html)
except Exception as e:
raise ValueError(f"{e.__class__.__name__} :: {e}, {self.html}")
self.meta = Haystack(html)
#print(json.dumps(self.meta, indent=4))
try:
if isinstance(self.html, str):
self.html = self.html.encode("utf-8")
self.lxml = lxml if lxml else parse_html(self.html)
except Exception as e:
raise ValueError(f"{e.__class__.__name__} :: {e}, {self.html}")
self.data = {
"content": self.content,
"url": self.url,
"title": self.title,
"published_at": self.published_at,
"description": self.summary,
"author": self.author,
"image_url": self.image_url,
"section": self.section,
"publisher": self.publisher,
"keywords": self.keywords,
"metadata": {k: v
for k, v in self.meta.data.items()},
}
self.data.update(
{k: row[k]
for k in self.passthrough_attrs if row and k in row})
def fix(text, lang, chars_rep, chars_pattern, punct_rep, punct_pattern):
global global_chars_lang
global_chars_lang = chars_rep
# htmlEntity=regex.compile(r'[&][[:space:]]*[#][[:space:]]*[0-9]{2,4}[[:space:]]*[;]?',regex.U)
chars3Re = regex.compile("[\uE000-\uFFFF]")
chars3Re2 = regex.compile("[\u2000-\u200F]")
chars3Re3 = regex.compile("\u007F|[\u0080-\u00A0]")
quotesRegex = regex.compile("(?P<start>[[:alpha:]])\'\'(?P<end>(s|S|t|T|m|M|d|D|re|RE|ll|LL|ve|VE|em|EM)\W)")
collapse_spaced_entities = regex.compile('([&][ ]*[#][ ]*)([0-9]{2,6})([ ]*[;])')
# Test encode: fix mojibake
ftfy_fixed_text = " ".join([ftfy.fix_text_segment(word, uncurl_quotes=False, fix_latin_ligatures=False) for word in text.split()])
# ftfy_fixed_text= ftfy.fix_text_segment(stripped_text, fix_entities=True,uncurl_quotes=False,fix_latin_ligatures=False)
# nicely_encoded_text = htmlEntity.sub(html.unescape, nicely_encoded_text)
nicely_encoded_text = html.unescape(ftfy_fixed_text)
# First replacing all HTML entities
# for substring in htmlEntity.findall(nicely_encoded_text):
# code=substring.replace(' ','')[2:].replace(';','')
# try:
# newChar=chr(int(code))
# except ValueError:
# newChar=code
# if newChar != "\n":
# nicely_encoded_text = nicely_encoded_text.replace(substring,newChar)
normalized_text = chars_pattern.sub(replace_chars, nicely_encoded_text)
if lang.lower() != "ja":
normalized_text = chars3Re.sub(replace_chars3, normalized_text)
normalized_text = chars3Re2.sub(replace_chars3, normalized_text)
normalized_text = chars3Re3.sub(replace_chars3, normalized_text)
normalized_text = quotesRegex.sub("\g<start>\'\g<end>", normalized_text)
normalized_text_with_normalized_punct = punct_pattern.sub(lambda m: punct_rep[re.escape(m.group(0))], normalized_text)
collapsed_spaces = re.sub('\s+', ' ', normalized_text_with_normalized_punct) # Collapse multiple spaces
collapsed_entities = collapse_spaced_entities.sub("&#\\2;", collapsed_spaces)
return collapsed_entities.strip(" \n")
def get_articles(feed, fn):
"""
Parse the specified feed,
gathering the latest new articles.
If an article matches one that already exists,
it is skipped.
The minimum length of an entry is
500 characters. Anything under will be ignored.
This will silently skip articles for which the full text
can't be retrieved (i.e. if it returns 404).
Some feeds, for whatever reason, do not include a `published`
date in their entry data. In which case, it is left as an
empty string.
Args:
| feed (Feed) -- the feed to fetch from.
| fn (Callable) -- function to use an article
"""
# Fetch the feed data.
data = feedparser.parse(feed.ext_url)
# If the `bozo` value is anything
# but 0, there was an error parsing (or connecting) to the feed.
if data.bozo:
# Some errors are ok.
if not isinstance(
data.bozo_exception,
feedparser.CharacterEncodingOverride) and not isinstance(
data.bozo_exception, feedparser.NonXMLContentType):
raise data.bozo_exception
for entry in data.entries:
# URL for this entry.
url = entry['links'][0]['href']
# Check for an existing Article.
# If one exists, skip.
if Article.query.filter_by(ext_url=url).count():
continue
# Complete HTML content for this entry.
try:
entry_data, html = extractor.extract_entry_data(url)
except (error.HTTPError, error.URLError, ConnectionResetError,
BadStatusLine) as e:
if type(e) == error.URLError or e.code == 404:
# Can't reach, skip.
logger.exception(
'Error extracting data for url {0}'.format(url))
continue
else:
# Just skip so things don't break!
logger.exception(
'Error extracting data for url {0}'.format(url))
continue
if entry_data is None:
continue
full_text = entry_data.cleaned_text
# Skip over entries that are too short.
if len(full_text) < 400:
continue
url = entry_data.canonical_link or url
published = parse(entry.get('published')) if entry.get(
'published') else entry_data.publish_date
updated = parse(
entry.get('updated')) if entry.get('updated') else published
title = entry.get('title', entry_data.title)
# Secondary check for an existing Article,
# by checking the title and source.
existing = Article.query.filter_by(title=title).first()
if existing and existing.source == feed.source:
continue
# Download and save the top article image.
image_url = extractor.extract_image(entry_data, filename=hash(url))
fn(
Article(ext_url=url,
source=feed.source,
feed=feed,
html=html,
text=fix_text_segment(full_text),
authors=extractor.extract_authors(entry),
tags=extractor.extract_tags(entry,
known_tags=entry_data.tags),
title=fix_text_segment(title),
created_at=published,
updated_at=updated,
image=image_url,
score=evaluator.score(url)))
def test_entities():
example = '&\n<html>\n&'
eq_(fix_text(example), '&\n<html>\n&')
eq_(fix_text_segment(example), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=True), '&\n<html>\n&')
eq_(fix_text(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment(example, fix_entities=False), '&\n<html>\n&')
eq_(fix_text_segment('<>', fix_entities=False), '<>')
eq_(fix_text_segment('<>', fix_entities=True), '<>')
eq_(fix_text_segment('<>'), '<>')
eq_(fix_text_segment('jednocześnie'), 'jednocześnie')
eq_(fix_text_segment('JEDNOCZEŚNIE'), 'JEDNOCZEŚNIE')
eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...')
eq_(fix_text_segment('ellipsis…', normalization='NFKC'), 'ellipsis...')
eq_(fix_text_segment('broken'), 'broken\x81')
eq_(unescape_html('euro €'), 'euro €')
eq_(unescape_html('not an entity x6;'), 'not an entity x6;')
def test_entities():
example = '&\n<html>\n&'
assert fix_text(example) == '&\n<html>\n&'
assert fix_text_segment(example) == '&\n<html>\n&'
assert fix_text(example, fix_entities=True) == '&\n<html>\n&'
assert fix_text_segment(example, fix_entities=True) == '&\n<html>\n&'
assert fix_text(example, fix_entities=False) == '&\n<html>\n&'
assert fix_text_segment(example,
fix_entities=False) == '&\n<html>\n&'
assert fix_text_segment('<>', fix_entities=False) == '<>'
assert fix_text_segment('<>', fix_entities=True) == '<>'
assert fix_text_segment('<>') == '<>'
assert fix_text_segment('jednocześnie') == 'jednocześnie'
assert fix_text_segment('JEDNOCZEŚNIE') == 'JEDNOCZEŚNIE'
assert fix_text_segment('ellipsis…',
normalization='NFKC') == 'ellipsis...'
assert fix_text_segment('ellipsis…',
normalization='NFKC') == 'ellipsis...'
assert fix_text_segment('broken') == 'broken\x81'
assert fix_text_segment('&amp;amp;') == '&'
assert unescape_html('euro €') == 'euro €'
assert unescape_html('EURO &EURO;') == 'EURO €'
assert unescape_html('not an entity x6;') == 'not an entity x6;'
assert unescape_html('JEDNOCZE&SACUTE;NIE') == 'JEDNOCZEŚNIE'
assert unescape_html('V&SCARON;ICHNI') == 'VŠICHNI'
assert unescape_html('') == ''
assert unescape_html('�') == '\ufffd'
assert (fix_text_segment('this is just informal english ¬ html') ==
'this is just informal english ¬ html')
def run(self, dumppath, use_patch):
if use_patch:
print('Patching out saving images to S3...')
patcher = patch('argos.util.storage.save_from_url', autospec=True, return_value='https://i.imgur.com/Zf9mXlj.jpg')
patcher.start()
else:
patcher = None
print('Loading sources...')
sources_map = {}
with open(os.path.join(dumppath, 'sources.json'), 'r') as f:
sources = json.load(f)
for i, s in enumerate(sources):
source = Source.query.filter(Source.name == s['name']).first()
if not source:
source = Source(name=s['name'])
db.session.add(source)
id = s['_id']['$oid']
sources_map[id] = source
progress_bar(i/(len(sources) - 1) * 100)
db.session.commit()
print('\nLoading feeds...')
feeds_map = {}
with open(os.path.join(dumppath, 'feeds.json'), 'r') as f:
feeds = json.load(f)
for i, f in enumerate(feeds):
feed = Feed.query.filter(Feed.ext_url == f['ext_url']).first()
if not feed:
feed = Feed(ext_url=f['ext_url'])
db.session.add(feed)
feed.source = sources_map[f['source']['$oid']]
id = f['_id']['$oid']
feeds_map[id] = feed
progress_bar(i/(len(feeds) - 1) * 100)
db.session.commit()
print('\nLoading articles...')
with open(os.path.join(dumppath, 'articles.json'), 'r') as f:
articles = json.load(f)
for i, a in enumerate(articles):
authors = []
for author in a['authors']:
authors.append(Author.find_or_create(name=author))
existing = Article.query.filter(Article.ext_url == a['ext_url']).first()
if not existing:
feed = feeds_map[a['feed']['$oid']]
article = Article(
ext_url=a['ext_url'],
source=feed.source,
feed=feed,
html=None, # not saved by argos.corpora
text=fix_text_segment(a['text']),
authors=authors,
tags=[],
title=fix_text_segment(a['title']),
created_at=datetime.fromtimestamp(a['created_at']['$date']/1000),
updated_at=datetime.fromtimestamp(a['updated_at']['$date']/1000),
image=a['image'],
score=evaluator.score(a['ext_url'])
)
db.session.add(article)
progress_bar(i/(len(articles) - 1) * 100)
print('Loaded {0} sources, {1} feeds, and {2} articles.'.format(len(sources), len(feeds), len(articles)))
print('Done!')
if patcher is not None:
patcher.stop()
