def exists(cls, text, language, is_raw=False): ''' Determine if a concept exists in ConceptNet. If `is_raw` is True, `text` is considered to be already in the raw (normalized) concept form. Otherwise, it is normalized before being checked in the database. ''' if not isinstance(language, Language): language = Language.get(language) if not is_raw: surface = SurfaceForm.get(text, language, False) if surface is not None: return True text = language.nl.normalize(text) return cls.exists_raw(text, language)
def get(cls, text, language, auto_create=False): """ Get the Concept represented by a given string of text. If the Concept does not exist, this method will return None by default. However, if the parameter ``auto_create=True`` is given, then this will create the Concept (adding it to the database) instead. You should not run the string through a normalizer, or use a string which came from :attr:`Concept.text` (which is equivalent). If you have a normalized string, you should use :meth:`get_raw` instead. """ if not isinstance(language, Language): language = Language.get(language) surface = SurfaceForm.get(text, language, auto_create) if surface is None: return Concept.get_raw(language.nl.normalize(text), language) return surface.concept
def get(text, lang, auto_create=False): if isinstance(lang, basestring): lang = Language.get(lang) nl = lang.nl try: known = SurfaceForm.objects.get(language=lang, text=text) return known except SurfaceForm.DoesNotExist: if not auto_create: return None else: lemma, residue = nl.lemma_factor(text) concept, created = Concept.objects.get_or_create(language=lang, text=lemma) if created: concept.save() # use get_or_create so it's atomic surface_form, _ = SurfaceForm.objects.get_or_create(concept=concept, text=text, residue=residue, language=lang) return surface_form
def read(self, request, lang): try: lang = Language.get(lang) return {'id': lang.id, 'sentence_count': lang.sentence_count} except Language.DoesNotExist: return rc.NOT_FOUND
# Create your views here. from models import UserTestQuestion, UserTestAnswer, VALUES from csc.corpus.models import Language from csc.conceptnet4.models import User from django.shortcuts import render_to_response import random en=Language.get('en') SOURCES = ['random_verbosity', 'cnet4', 'verbosity'] human_names_jan09 = { 'random': 'Randomly generated statements', 'cnet35': 'Statements people have contributed to ConceptNet', 'cnet35_svd': 'Statements predicted to be true using ConceptNet', 'cnet35_wordnet_svd': 'Statements predicted to be true by combining ConceptNet and WordNet', } human_names = { 'random_verbosity': 'Randomly generated statements', 'cnet4': 'Statements people have contributed to ConceptNet', 'verbosity': 'Statements people have contributed using Verbosity' } def inferences_from_file(filename): f = open(filename) got = [] for line in file: line = line.strip() if not line: continue got.append(eval(line)) return got
# Raise again raise e # Process sentences page_range = [p for p in paginator.page_range if p >= start_page] for i in page_range: sentences = paginator.page(i).object_list # Update progress batch.status = "process_sentence_batch " + str(i) + "/" + str(paginator.num_pages) batch.progress_num = i batch.progress_den = paginator.num_pages batch.save() try: do_batch(sentences) except Exception, e: #improbable exception for now batch.status = "process_sentence_batch " + str(i) + "/" + str(paginator.num_pages) + " ERROR!" batch.remarks = str(e.sentence) + "\n" + str(e) + "\n" + e.tb print "***TRACEBACK***" print batch.remarks batch.save() raise e if __name__ == '__main__': user = User.objects.get(username='******') lang = Language.get('en') run(user, lang, start_page=50000)
def get_language(lang): try: return Language.get(lang) except Language.DoesNotExist: raise Http404