def exists(cls, text, language, is_raw=False):
'''
Determine if a concept exists in ConceptNet.
If `is_raw` is True, `text` is considered to be already in the
raw (normalized) concept form. Otherwise, it is normalized
before being checked in the database.
'''
if not isinstance(language, Language):
language = Language.get(language)
if not is_raw:
surface = SurfaceForm.get(text, language, False)
if surface is not None: return True
text = language.nl.normalize(text)
return cls.exists_raw(text, language)
def get(cls, text, language, auto_create=False):
"""
Get the Concept represented by a given string of text.
If the Concept does not exist, this method will return None by default.
However, if the parameter ``auto_create=True`` is given, then this will
create the Concept (adding it to the database) instead.
You should not run the string through a normalizer, or use a string
which came from :attr:`Concept.text` (which is equivalent). If you
have a normalized string, you should use :meth:`get_raw` instead.
"""
if not isinstance(language, Language):
language = Language.get(language)
surface = SurfaceForm.get(text, language, auto_create)
if surface is None:
return Concept.get_raw(language.nl.normalize(text), language)
return surface.concept
def get(text, lang, auto_create=False):
if isinstance(lang, basestring):
lang = Language.get(lang)
nl = lang.nl
try:
known = SurfaceForm.objects.get(language=lang, text=text)
return known
except SurfaceForm.DoesNotExist:
if not auto_create:
return None
else:
lemma, residue = nl.lemma_factor(text)
concept, created = Concept.objects.get_or_create(language=lang, text=lemma)
if created: concept.save()
# use get_or_create so it's atomic
surface_form, _ = SurfaceForm.objects.get_or_create(concept=concept,
text=text, residue=residue, language=lang)
return surface_form
def read(self, request, lang):
try:
lang = Language.get(lang)
return {'id': lang.id, 'sentence_count': lang.sentence_count}
except Language.DoesNotExist:
return rc.NOT_FOUND
# Create your views here.
from models import UserTestQuestion, UserTestAnswer, VALUES
from csc.corpus.models import Language
from csc.conceptnet4.models import User
from django.shortcuts import render_to_response
import random
en=Language.get('en')
SOURCES = ['random_verbosity', 'cnet4', 'verbosity']
human_names_jan09 = {
'random': 'Randomly generated statements',
'cnet35': 'Statements people have contributed to ConceptNet',
'cnet35_svd': 'Statements predicted to be true using ConceptNet',
'cnet35_wordnet_svd':
'Statements predicted to be true by combining ConceptNet and WordNet',
}
human_names = {
'random_verbosity': 'Randomly generated statements',
'cnet4': 'Statements people have contributed to ConceptNet',
'verbosity': 'Statements people have contributed using Verbosity'
}
def inferences_from_file(filename):
f = open(filename)
got = []
for line in file:
line = line.strip()
if not line: continue
got.append(eval(line))
return got
# Raise again
raise e
# Process sentences
page_range = [p for p in paginator.page_range if p >= start_page]
for i in page_range:
sentences = paginator.page(i).object_list
# Update progress
batch.status = "process_sentence_batch " + str(i) + "/" + str(paginator.num_pages)
batch.progress_num = i
batch.progress_den = paginator.num_pages
batch.save()
try: do_batch(sentences)
except Exception, e: #improbable exception for now
batch.status = "process_sentence_batch " + str(i) + "/" + str(paginator.num_pages) + " ERROR!"
batch.remarks = str(e.sentence) + "\n" + str(e) + "\n" + e.tb
print "***TRACEBACK***"
print batch.remarks
batch.save()
raise e
if __name__ == '__main__':
user = User.objects.get(username='******')
lang = Language.get('en')
run(user, lang, start_page=50000)
def get_language(lang):
try:
return Language.get(lang)
except Language.DoesNotExist:
raise Http404
