def simtitle( request ):
"""calculate similarity based on title and naive threshold"""
n = NGram( warp=WARP, iconv=enrich, key=lambda x: x.title )
articles = Article.objects.filter( status = "live" ).order_by( "date_published" )[:1000]
results = []
for article in articles:
article.is_duplicate = False
article.duplicate_of = None
article.save()
sim = filter( lambda a: a[1] >= 0.4, n.search( article.title ) )
for match in sim:
nearest = match[0]
if nearest.is_duplicate:
nearest = nearest.duplicate_of
if NGram.compare( article.title, nearest.title ) < 0.7:
results.append( article )
break
article.is_duplicate = True
article.duplicate_of = nearest
article.save()
break
else:
results.append( article )
n.add( article )
return render( request, "dump.html", dictionary = { "article_list": results, } )
def build_title_index(movies, tvshows):
start = time.time()
entities = list(itertools.chain.from_iterable([movies, tvshows]))
values = [entity['title'] for entity in entities]
mapped_entities = {}
for entity in entities:
value = entity['title']
if value not in mapped_entities:
mapped_entities[value] = []
mapped_entities[value].append(entity)
logger.debug('Iterating title took {} ms'.format(
int((time.time() - start) * 1000)))
start = time.time()
index = NGram()
for value in values:
index.add(value)
logger.debug('Building title index took {} ms'.format(
int((time.time() - start) * 1000)))
return index, mapped_entities
def simtitle(request):
"""calculate similarity based on title and naive threshold"""
n = NGram(warp=2.5, iconv=enrich)
articles = Article.objects.filter(
status="live").order_by("date_published")[:1000]
results = []
for article in articles:
article.is_duplicate = False
article.duplicate_of = None
article.save()
sim = filter(lambda a: a[1] >= 0.7, n.search(article.title))
for match in sim:
nearest = match[0]
if nearest.is_duplicate:
nearest = nearest.duplicate_of
if NGram.compare(article.title, nearest.title) < 0.7:
results.append(article)
break
article.is_duplicate = True
article.duplicate_of = nearest
article.save()
break
else:
results.append(article)
n.add(article)
return render(request, "dump.html", dictionary={
"article_list": results,
})
def build_cast_index(movies, tvshows, key):
start = time.time()
entities = list(itertools.chain.from_iterable([movies, tvshows]))
values = [[cast[key] for cast in entity['cast']] for entity in entities]
values = list(set(itertools.chain.from_iterable(values)))
mapped_entities = {}
for entity in entities:
for cast in entity['cast']:
value = cast[key]
if value not in mapped_entities:
mapped_entities[value] = []
mapped_entities[value].append(entity)
logger.debug('Iterating {} took {} ms'.format(
key, int((time.time() - start) * 1000)))
start = time.time()
index = NGram()
for value in values:
index.add(value)
logger.debug('Building {} index took {} ms'.format(
key, int((time.time() - start) * 1000)))
return index, mapped_entities
def build_collection_index(movies, tvshows):
start = time.time()
entities = list(itertools.chain.from_iterable([movies, tvshows]))
values = list(
set([
parse_collection(entity['set']) for entity in entities
if 'set' in entity and len(entity['set']) > 0
]))
mapped_entities = {}
for entity in entities:
if 'set' in entity and entity['set']:
value = parse_collection(entity['set'])
if value not in mapped_entities:
mapped_entities[value] = []
mapped_entities[value].append(entity)
logger.debug('Iterating collection took {} ms'.format(
int((time.time() - start) * 1000)))
start = time.time()
index = NGram()
for value in values:
index.add(value)
logger.debug('Building collection index took {} ms'.format(
int((time.time() - start) * 1000)))
return index, mapped_entities
def build_multiclusters(inlines, threshold=0.05, N=4):
clusters = []
ignoreus = []
for i, iline in enumerate(inlines):
if i in ignoreus:
continue
iString = " ".join(iline.split(" :::: ")[:3])
ignoreus.append(i)
icluster = {}
icluster[iline] = -1
iModel = NGram(iString)
for j in range(i, len(inlines)):
if j in ignoreus:
continue
jline = inlines[j]
jString = " ".join(jline.split(" :::: ")[:3])
results = iModel.search(jString)
score = sum([y for x,y in results]) / len(results) \
if len(results) > 0 else 0.0
print score
if score > threshold:
icluster[jline] = score
iModel.add(jString)
ignoreus.append(j)
clusters.append(icluster)
return clusters
class Plagiarism:
def __init__(self,text):
self.ng=NGram()
file = open(text,"r")
linea = file.readline()
while linea != '':
if linea != '\n':
self.ng.add(linea)
linea = file.readline()
self.lsn=list(self.ng);
file.close()
def verify(self,text_compare):
results = []
dictio = []
file2 = open(text_compare,"r")
linea2 = file2.readline()
while linea2 != '':
if linea2 != '\n':
dictio += [self.ng.items_sharing_ngrams(linea2)]
compares = 0.0
for parrafo in self.lsn:
comp = NGram.compare(parrafo,linea2)
if compares < comp:
compares = comp
results += [compares]
linea2 = file2.readline()
file2.close()
major_ocurrences=[]
for d in dictio:
major=0
for val in d.values():
if major<val:
major=val
major_ocurrences+=[major]
avg_perc=0.0
for r in results:
avg_perc+=r
avg_perc=avg_perc/len(results)
print("Mayor numero de ocurrencias por parrafo del texto copia: "+repr(major_ocurrences))
print("Porcentaje Similitud: "+repr(avg_perc))
def handle( self, *args, **options ):
if "simonly" in args:
new_count = 100000
else:
new_count = 0
for source in Source.objects.filter( scraper = 'feedparser', status__in = ( 'silent', 'live' ) ):
l = feedparser.parse( source.scraper_config )
ok = True
if l[ "bozo" ] == 1:
if not isinstance( l[ "bozo_exception" ], feedparser.ThingsNobodyCaresAboutButMe ):
ok = False
if ok:
for article in l[ "entries" ]:
#print "Reading feed entry %s: '%s'" % ( article[ "id" ], article[ "title" ] )
a, created = Article.objects.get_or_create(
source = source,
# Wordpress RSS IDs are unique internet-wide, and are immutable (unlike URLs)
source_reference = article[ "id" ],
defaults = {
'date_created' : datetime.now(),
'source_url' : article[ "link" ],
'title' : self.normalise( article[ "title" ] ),
'num_comments' : article.get( "slash_comments", 0 ),
'summary' : article[ "summary" ],
'author' : article.get( "author", "" ),
'date_published' : datetime(*(article[ "updated_parsed" ][:6])),
'status' : "live"
}
)
if created:
#print "Creating new article."
pass
else:
#print "Updating article."
pass
new_count += 1
if article.has_key( "content" ):
# TODO test for multiple content blocks and pick most appropriate
a.body = article[ "content" ][0][ "value" ]
a.tags.clear()
for tag in article.get( "tags", () ):
a.tags.add( tag[ "term" ] )
a.save()
else:
logging.error( "Could not read feed for file '%s': %s" % ( source.scraper_config, l[ "bozo_exception" ] ) )
logging.error( "Skipping '%s': %s" % ( source.scraper_config, l[ "bozo_exception" ] ) )
break
#calculate similarities
#create a similarity corpus of last 200 docs
def enrich( obj ):
s = unicode( obj )
# simple stop words
s = re.sub( r"\b(the|of|in|a)\b", "", s, re.IGNORECASE )
# type prefixes
s = re.sub( r"^(trailer|review|report|screenshots|video):\s*", "", s, re.IGNORECASE )
return s
n = NGram( warp=2.5, iconv=enrich )
articles = Article.objects.filter( status = "live" ).order_by( "date_published" )[:(new_count*4)]
for article in articles:
if "simonly" in args:
article.is_duplicate = False
article.duplicate_of = None
article.save()
continue
#articles = Article.objects.filter( status = "live", is_duplicate = False ).order_by( "-date_published" )[:new_count]
#for article in articles:
#print( u"similarity for %s" % ( article.title, ) )
sim = filter( lambda a: a[1] > 0.4, n.search( article.title ) )
for match in sim:
nearest = match[0]
if nearest.source == article.source:
continue
if nearest.is_duplicate:
nearest = nearest.duplicate_of
# do it again!
if nearest.source == article.source:
continue
article.is_duplicate = True
article.duplicate_of = nearest
#print u" is duplicate of %s" % ( nearest.title, )
article.save()
break
n.add( article )
import csv
from ngram import NGram
records = NGram()
with open('./data/houses.csv', 'r', encoding='windows-1251') as f:
for line in csv.reader(f, delimiter=';'):
records.add(' '.join(list(line)).lower())
while True:
print('Enter search text:')
search_text = input().lower()
print('find', records.find(search_text), 0.8)
def test_unigram(self):
n = NGram(0)
n.add('after')
n.next_word() == 'after'
def test_trigram(self):
n = NGram(2)
n.add('after', ('before', 'other'))
assert n.next_word(('before', 'other')) == 'after'
def test_bigram(self):
n = NGram(1)
n.add('after', ('before'))
assert n.next_word(('before')) == 'after'
def handle(self, *args, **options):
if "simonly" in args:
new_count = 100000
else:
new_count = 0
for source in Source.objects.filter(scraper='feedparser',
status__in=('silent', 'live')):
l = feedparser.parse(source.scraper_config)
ok = True
if l["bozo"] == 1:
if not isinstance(l["bozo_exception"],
feedparser.ThingsNobodyCaresAboutButMe):
ok = False
if ok:
for article in l["entries"]:
#print "Reading feed entry %s: '%s'" % ( article[ "id" ], article[ "title" ] )
a, created = Article.objects.get_or_create(
source=source,
# Wordpress RSS IDs are unique internet-wide, and are immutable (unlike URLs)
source_reference=article["id"],
defaults={
'date_created':
datetime.now(),
'source_url':
article["link"],
'title':
self.normalise(article["title"]),
'num_comments':
article.get("slash_comments", 0),
'summary':
article["summary"],
'author':
article.get("author", ""),
'date_published':
datetime(*(article["updated_parsed"][:6])),
'status':
"live"
})
if created:
#print "Creating new article."
pass
else:
#print "Updating article."
pass
new_count += 1
if article.has_key("content"):
# TODO test for multiple content blocks and pick most appropriate
a.body = article["content"][0]["value"]
a.tags.clear()
for tag in article.get("tags", ()):
a.tags.add(tag["term"])
a.save()
else:
logging.error("Could not read feed for file '%s': %s" %
(source.scraper_config, l["bozo_exception"]))
logging.error("Skipping '%s': %s" %
(source.scraper_config, l["bozo_exception"]))
break
#calculate similarities
#create a similarity corpus of last 200 docs
def enrich(obj):
s = unicode(obj)
# simple stop words
s = re.sub(r"\b(the|of|in|a)\b", "", s, re.IGNORECASE)
# type prefixes
s = re.sub(r"^(trailer|review|report|screenshots|video):\s*", "",
s, re.IGNORECASE)
return s
n = NGram(warp=2.5, iconv=enrich)
articles = Article.objects.filter(
status="live").order_by("date_published")[:(new_count * 4)]
for article in articles:
if "simonly" in args:
article.is_duplicate = False
article.duplicate_of = None
article.save()
continue
#articles = Article.objects.filter( status = "live", is_duplicate = False ).order_by( "-date_published" )[:new_count]
#for article in articles:
#print( u"similarity for %s" % ( article.title, ) )
sim = filter(lambda a: a[1] > 0.4, n.search(article.title))
for match in sim:
nearest = match[0]
if nearest.source == article.source:
continue
if nearest.is_duplicate:
nearest = nearest.duplicate_of
# do it again!
if nearest.source == article.source:
continue
article.is_duplicate = True
article.duplicate_of = nearest
#print u" is duplicate of %s" % ( nearest.title, )
article.save()
break
n.add(article)
def test_unigram(self):
n = NGram(0)
n.add('after')
n.next_word() == 'after'
def test_trigram(self):
n = NGram(2)
n.add('after', ('before', 'other'))
assert n.next_word(('before', 'other')) == 'after'
def test_bigram(self):
n = NGram(1)
n.add('after', ('before'))
assert n.next_word(('before')) == 'after'
